feat: add batch auto-commit mode for Dolt to reduce commit bloat
Add a "batch" auto-commit mode that defers DOLT_COMMIT until logical boundaries (bd sync, bd dolt commit) instead of creating a commit after every bd CLI invocation. This addresses the 49K commit bloat problem reported by the Dolt team. Changes: - Add doltAutoCommitBatch mode alongside existing on/off modes - Change embedded mode default from "on" to "batch" - Add CommitPending method to DoltStore that generates descriptive summary messages by querying dolt_diff (e.g. "3 created, 2 updated") - Wire batch commit into bd sync and bd dolt commit - Update existing tests and add new tests for batch mode Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com> Executed-By: beads/polecats/obsidian Rig: beads Role: polecats
This commit is contained in:
parent
b13720ddb1
commit
836835d8f6
9 changed files with 323 additions and 21 deletions
|
|
@ -203,9 +203,12 @@ var doltCommitCmd = &cobra.Command{
|
|||
Short: "Create a Dolt commit from pending changes",
|
||||
Long: `Create a Dolt commit from any uncommitted changes in the working set.
|
||||
|
||||
This is useful before push operations that require a clean working set.
|
||||
Normally, auto-commit handles this after each bd write command, but manual
|
||||
commit may be needed if auto-commit was off or changes were made externally.
|
||||
This is the primary commit point for batch mode. When auto-commit is set to
|
||||
"batch", changes accumulate in the working set across multiple bd commands and
|
||||
are committed together here with a descriptive summary message.
|
||||
|
||||
Also useful before push operations that require a clean working set, or when
|
||||
auto-commit was off or changes were made externally.
|
||||
|
||||
For more options (--stdin, custom messages), see: bd vc commit`,
|
||||
Run: func(cmd *cobra.Command, args []string) {
|
||||
|
|
@ -217,17 +220,29 @@ For more options (--stdin, custom messages), see: bd vc commit`,
|
|||
}
|
||||
msg, _ := cmd.Flags().GetString("message")
|
||||
if msg == "" {
|
||||
msg = "bd: manual commit (dolt commit)"
|
||||
}
|
||||
if err := st.Commit(ctx, msg); err != nil {
|
||||
errLower := strings.ToLower(err.Error())
|
||||
if strings.Contains(errLower, "nothing to commit") || strings.Contains(errLower, "no changes") {
|
||||
// No explicit message — use CommitPending which generates a
|
||||
// descriptive summary of accumulated changes.
|
||||
committed, err := st.CommitPending(ctx, getActor())
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Error: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
if !committed {
|
||||
fmt.Println("Nothing to commit.")
|
||||
return
|
||||
}
|
||||
fmt.Fprintf(os.Stderr, "Error: %v\n", err)
|
||||
os.Exit(1)
|
||||
} else {
|
||||
if err := st.Commit(ctx, msg); err != nil {
|
||||
errLower := strings.ToLower(err.Error())
|
||||
if strings.Contains(errLower, "nothing to commit") || strings.Contains(errLower, "no changes") {
|
||||
fmt.Println("Nothing to commit.")
|
||||
return
|
||||
}
|
||||
fmt.Fprintf(os.Stderr, "Error: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
commandDidExplicitDoltCommit = true
|
||||
fmt.Println("Committed.")
|
||||
},
|
||||
}
|
||||
|
|
|
|||
|
|
@ -19,7 +19,9 @@ type doltAutoCommitParams struct {
|
|||
// maybeAutoCommit creates a Dolt commit after a successful write command when enabled.
|
||||
//
|
||||
// Semantics:
|
||||
// - Only applies when dolt auto-commit is enabled (on) AND the active store is versioned (Dolt).
|
||||
// - Only applies when dolt auto-commit is "on" AND the active store is versioned (Dolt).
|
||||
// - In "batch" mode, commits are deferred — changes accumulate in the working set
|
||||
// until an explicit commit point (bd sync, bd dolt commit).
|
||||
// - Uses Dolt's "commit all" behavior under the hood (DOLT_COMMIT -Am).
|
||||
// - Treats "nothing to commit" as a no-op.
|
||||
func maybeAutoCommit(ctx context.Context, p doltAutoCommitParams) error {
|
||||
|
|
@ -27,6 +29,8 @@ func maybeAutoCommit(ctx context.Context, p doltAutoCommitParams) error {
|
|||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// In batch mode, skip per-command commits. Changes stay in the working set
|
||||
// and are committed at logical boundaries (bd sync, bd dolt commit).
|
||||
if mode != doltAutoCommitOn {
|
||||
return nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -8,8 +8,9 @@ import (
|
|||
type doltAutoCommitMode string
|
||||
|
||||
const (
|
||||
doltAutoCommitOff doltAutoCommitMode = "off"
|
||||
doltAutoCommitOn doltAutoCommitMode = "on"
|
||||
doltAutoCommitOff doltAutoCommitMode = "off"
|
||||
doltAutoCommitOn doltAutoCommitMode = "on"
|
||||
doltAutoCommitBatch doltAutoCommitMode = "batch"
|
||||
)
|
||||
|
||||
func getDoltAutoCommitMode() (doltAutoCommitMode, error) {
|
||||
|
|
@ -24,7 +25,9 @@ func getDoltAutoCommitMode() (doltAutoCommitMode, error) {
|
|||
return doltAutoCommitOff, nil
|
||||
case doltAutoCommitOn:
|
||||
return doltAutoCommitOn, nil
|
||||
case doltAutoCommitBatch:
|
||||
return doltAutoCommitBatch, nil
|
||||
default:
|
||||
return "", fmt.Errorf("invalid --dolt-auto-commit=%q (valid: off, on)", doltAutoCommit)
|
||||
return "", fmt.Errorf("invalid --dolt-auto-commit=%q (valid: off, on, batch)", doltAutoCommit)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -122,8 +122,8 @@ func TestDoltAutoCommit_On_WritesAdvanceHead(t *testing.T) {
|
|||
|
||||
before := doltHeadCommit(t, tmpDir, env)
|
||||
|
||||
// A write command should create a new Dolt commit (auto-commit default is on in embedded mode).
|
||||
out, err := runBDExecAllowErrorWithEnv(t, tmpDir, env, "create", "Auto-commit test", "--json")
|
||||
// Explicitly enable auto-commit=on (default is now "batch" in embedded mode).
|
||||
out, err := runBDExecAllowErrorWithEnv(t, tmpDir, env, "--dolt-auto-commit", "on", "create", "Auto-commit test", "--json")
|
||||
if err != nil {
|
||||
t.Fatalf("bd create failed: %v\n%s", err, out)
|
||||
}
|
||||
|
|
@ -148,7 +148,7 @@ func TestDoltAutoCommit_On_WritesAdvanceHead(t *testing.T) {
|
|||
}
|
||||
|
||||
// A read-only command should not create another commit.
|
||||
out, err = runBDExecAllowErrorWithEnv(t, tmpDir, env, "list")
|
||||
out, err = runBDExecAllowErrorWithEnv(t, tmpDir, env, "--dolt-auto-commit", "on", "list")
|
||||
if err != nil {
|
||||
t.Fatalf("bd list failed: %v\n%s", err, out)
|
||||
}
|
||||
|
|
@ -158,6 +158,66 @@ func TestDoltAutoCommit_On_WritesAdvanceHead(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
func TestDoltAutoCommit_Batch_DefersCommit(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping slow integration test in short mode")
|
||||
}
|
||||
if runtime.GOOS == windowsOS {
|
||||
t.Skip("dolt integration test not supported on windows")
|
||||
}
|
||||
|
||||
tmpDir := createTempDirWithCleanup(t)
|
||||
setupGitRepoForIntegration(t, tmpDir)
|
||||
|
||||
env := []string{
|
||||
"BEADS_TEST_MODE=1",
|
||||
"BEADS_NO_DAEMON=1",
|
||||
}
|
||||
|
||||
initOut, initErr := runBDExecAllowErrorWithEnv(t, tmpDir, env, "init", "--backend", "dolt", "--prefix", "test", "--quiet")
|
||||
if initErr != nil {
|
||||
if isDoltBackendUnavailable(initOut) {
|
||||
t.Skipf("dolt backend not available: %s", initOut)
|
||||
}
|
||||
t.Fatalf("bd init --backend dolt failed: %v\n%s", initErr, initOut)
|
||||
}
|
||||
|
||||
before := doltHeadCommit(t, tmpDir, env)
|
||||
|
||||
// In batch mode (default for embedded), writes should NOT advance HEAD.
|
||||
out, err := runBDExecAllowErrorWithEnv(t, tmpDir, env, "--dolt-auto-commit", "batch", "create", "Batch test 1", "--json")
|
||||
if err != nil {
|
||||
t.Fatalf("bd create failed: %v\n%s", err, out)
|
||||
}
|
||||
|
||||
afterCreate := doltHeadCommit(t, tmpDir, env)
|
||||
if afterCreate != before {
|
||||
t.Fatalf("expected Dolt HEAD unchanged in batch mode; before=%s after=%s", before, afterCreate)
|
||||
}
|
||||
|
||||
// Create another issue — still deferred.
|
||||
out, err = runBDExecAllowErrorWithEnv(t, tmpDir, env, "--dolt-auto-commit", "batch", "create", "Batch test 2", "--json")
|
||||
if err != nil {
|
||||
t.Fatalf("bd create (2) failed: %v\n%s", err, out)
|
||||
}
|
||||
|
||||
afterCreate2 := doltHeadCommit(t, tmpDir, env)
|
||||
if afterCreate2 != before {
|
||||
t.Fatalf("expected Dolt HEAD still unchanged; before=%s after=%s", before, afterCreate2)
|
||||
}
|
||||
|
||||
// An explicit "bd dolt commit" should commit all accumulated changes.
|
||||
out, err = runBDExecAllowErrorWithEnv(t, tmpDir, env, "--dolt-auto-commit", "batch", "dolt", "commit")
|
||||
if err != nil {
|
||||
t.Fatalf("bd dolt commit failed: %v\n%s", err, out)
|
||||
}
|
||||
|
||||
afterCommit := doltHeadCommit(t, tmpDir, env)
|
||||
if afterCommit == before {
|
||||
t.Fatalf("expected Dolt HEAD to advance after explicit commit; before=%s after=%s", before, afterCommit)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDoltAutoCommit_Off_DoesNotAdvanceHead(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping slow integration test in short mode")
|
||||
|
|
|
|||
|
|
@ -38,3 +38,43 @@ func TestIsDoltNothingToCommit(t *testing.T) {
|
|||
t.Fatal("unexpected classification")
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetDoltAutoCommitMode_Batch(t *testing.T) {
|
||||
old := doltAutoCommit
|
||||
defer func() { doltAutoCommit = old }()
|
||||
|
||||
doltAutoCommit = "batch"
|
||||
mode, err := getDoltAutoCommitMode()
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if mode != doltAutoCommitBatch {
|
||||
t.Fatalf("expected batch, got %q", mode)
|
||||
}
|
||||
|
||||
// Also verify the other modes still work
|
||||
doltAutoCommit = "on"
|
||||
mode, err = getDoltAutoCommitMode()
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if mode != doltAutoCommitOn {
|
||||
t.Fatalf("expected on, got %q", mode)
|
||||
}
|
||||
|
||||
doltAutoCommit = "off"
|
||||
mode, err = getDoltAutoCommitMode()
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if mode != doltAutoCommitOff {
|
||||
t.Fatalf("expected off, got %q", mode)
|
||||
}
|
||||
|
||||
// Invalid mode
|
||||
doltAutoCommit = "invalid"
|
||||
_, err = getDoltAutoCommitMode()
|
||||
if err == nil {
|
||||
t.Fatal("expected error for invalid mode")
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -193,7 +193,7 @@ func init() {
|
|||
rootCmd.PersistentFlags().BoolVar(&sandboxMode, "sandbox", false, "Sandbox mode: disables auto-sync")
|
||||
rootCmd.PersistentFlags().BoolVar(&allowStale, "allow-stale", false, "Allow operations on potentially stale data (skip staleness check)")
|
||||
rootCmd.PersistentFlags().BoolVar(&readonlyMode, "readonly", false, "Read-only mode: block write operations (for worker sandboxes)")
|
||||
rootCmd.PersistentFlags().StringVar(&doltAutoCommit, "dolt-auto-commit", "", "Dolt backend: auto-commit after write commands (off|on). Default: on for embedded, off for server mode. Override via config key dolt.auto-commit")
|
||||
rootCmd.PersistentFlags().StringVar(&doltAutoCommit, "dolt-auto-commit", "", "Dolt backend: auto-commit after write commands (off|on|batch). Default: batch for embedded, off for server mode. Override via config key dolt.auto-commit")
|
||||
rootCmd.PersistentFlags().BoolVar(&profileEnabled, "profile", false, "Generate CPU profile for performance analysis")
|
||||
rootCmd.PersistentFlags().BoolVarP(&verboseFlag, "verbose", "v", false, "Enable verbose/debug output")
|
||||
rootCmd.PersistentFlags().BoolVarP(&quietFlag, "quiet", "q", false, "Suppress non-essential output (errors only)")
|
||||
|
|
@ -539,13 +539,15 @@ var rootCmd = &cobra.Command{
|
|||
// config explicitly set it. Server mode defaults to OFF because the
|
||||
// server handles commits via its own transaction lifecycle; firing
|
||||
// DOLT_COMMIT after every write under concurrent load causes
|
||||
// 'database is read only' errors. Embedded mode defaults to ON so
|
||||
// each write is durably committed.
|
||||
// 'database is read only' errors. Embedded mode defaults to BATCH
|
||||
// so that multiple operations accumulate in the working set and are
|
||||
// committed together at logical boundaries (bd sync, bd dolt commit),
|
||||
// reducing commit bloat from per-command commits.
|
||||
if strings.TrimSpace(doltAutoCommit) == "" {
|
||||
if doltCfg.ServerMode {
|
||||
doltAutoCommit = string(doltAutoCommitOff)
|
||||
} else {
|
||||
doltAutoCommit = string(doltAutoCommitOn)
|
||||
doltAutoCommit = string(doltAutoCommitBatch)
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -40,6 +40,18 @@ For data interchange:
|
|||
return
|
||||
}
|
||||
|
||||
// Batch commit: if auto-commit mode is "batch", commit all pending
|
||||
// changes as a single logical Dolt commit before sync operations.
|
||||
// This is the primary commit boundary for batch mode.
|
||||
if mode, err := getDoltAutoCommitMode(); err == nil && mode == doltAutoCommitBatch {
|
||||
if _, commitErr := store.CommitPending(rootCtx, getActor()); commitErr != nil {
|
||||
fmt.Fprintf(os.Stderr, "Warning: batch commit failed: %v\n", commitErr)
|
||||
}
|
||||
}
|
||||
|
||||
// Mark as explicit commit so PersistentPostRun doesn't double-commit
|
||||
commandDidExplicitDoltCommit = true
|
||||
|
||||
// In dolt-native mode, skip JSONL export — Dolt is the source of truth.
|
||||
// Only push to Dolt remote if configured.
|
||||
if config.GetSyncMode() == config.SyncModeDoltNative {
|
||||
|
|
|
|||
|
|
@ -709,6 +709,80 @@ func (s *DoltStore) Commit(ctx context.Context, message string) error {
|
|||
return nil
|
||||
}
|
||||
|
||||
// CommitPending creates a single Dolt commit for all uncommitted changes in the working set.
|
||||
// Returns (true, nil) if changes were committed, (false, nil) if there was nothing to commit,
|
||||
// or (false, err) on failure. The commit message summarizes the accumulated changes by
|
||||
// querying dolt_diff to count issue-level operations.
|
||||
//
|
||||
// This is the primary commit mechanism for batch mode, where multiple bd commands
|
||||
// accumulate changes in the working set before committing at a logical boundary.
|
||||
func (s *DoltStore) CommitPending(ctx context.Context, actor string) (bool, error) {
|
||||
// Check if there are any uncommitted changes
|
||||
status, err := s.Status(ctx)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("failed to check status: %w", err)
|
||||
}
|
||||
if len(status.Staged) == 0 && len(status.Unstaged) == 0 {
|
||||
return false, nil // Nothing to commit
|
||||
}
|
||||
|
||||
msg := s.buildBatchCommitMessage(ctx, actor)
|
||||
if err := s.Commit(ctx, msg); err != nil {
|
||||
return false, err
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// buildBatchCommitMessage generates a descriptive commit message summarizing
|
||||
// what changed since the last commit by querying dolt_diff against HEAD.
|
||||
func (s *DoltStore) buildBatchCommitMessage(ctx context.Context, actor string) string {
|
||||
if actor == "" {
|
||||
actor = s.committerName
|
||||
}
|
||||
|
||||
// Count issue-level changes by diff type
|
||||
var added, modified, removed int
|
||||
rows, err := s.db.QueryContext(ctx, `
|
||||
SELECT diff_type, COUNT(*) as cnt
|
||||
FROM dolt_diff('HEAD', 'WORKING', 'issues')
|
||||
GROUP BY diff_type
|
||||
`)
|
||||
if err == nil {
|
||||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
var diffType string
|
||||
var count int
|
||||
if scanErr := rows.Scan(&diffType, &count); scanErr == nil {
|
||||
switch diffType {
|
||||
case "added":
|
||||
added = count
|
||||
case "modified":
|
||||
modified = count
|
||||
case "removed":
|
||||
removed = count
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Build descriptive message
|
||||
var parts []string
|
||||
if added > 0 {
|
||||
parts = append(parts, fmt.Sprintf("%d created", added))
|
||||
}
|
||||
if modified > 0 {
|
||||
parts = append(parts, fmt.Sprintf("%d updated", modified))
|
||||
}
|
||||
if removed > 0 {
|
||||
parts = append(parts, fmt.Sprintf("%d deleted", removed))
|
||||
}
|
||||
|
||||
if len(parts) == 0 {
|
||||
return fmt.Sprintf("bd: batch commit by %s", actor)
|
||||
}
|
||||
return fmt.Sprintf("bd: batch commit by %s — %s", actor, strings.Join(parts, ", "))
|
||||
}
|
||||
|
||||
// Push pushes commits to the remote.
|
||||
// When remote credentials are configured (for Hosted Dolt), sets DOLT_REMOTE_PASSWORD
|
||||
// env var and passes --user flag to authenticate.
|
||||
|
|
|
|||
|
|
@ -3,7 +3,10 @@
|
|||
package dolt
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/steveyegge/beads/internal/types"
|
||||
)
|
||||
|
||||
// TestCommitExists tests the CommitExists method.
|
||||
|
|
@ -83,3 +86,92 @@ func TestCommitExists(t *testing.T) {
|
|||
}
|
||||
})
|
||||
}
|
||||
|
||||
// TestCommitPending tests the batch commit mechanism.
|
||||
func TestCommitPending(t *testing.T) {
|
||||
store, cleanup := setupTestStore(t)
|
||||
defer cleanup()
|
||||
|
||||
ctx, cancel := testContext(t)
|
||||
defer cancel()
|
||||
|
||||
// Initial commit so the store has a clean HEAD
|
||||
if err := store.Commit(ctx, "initial state"); err != nil {
|
||||
t.Fatalf("initial commit failed: %v", err)
|
||||
}
|
||||
|
||||
t.Run("returns false when nothing to commit", func(t *testing.T) {
|
||||
committed, err := store.CommitPending(ctx, "test-actor")
|
||||
if err != nil {
|
||||
t.Fatalf("CommitPending failed: %v", err)
|
||||
}
|
||||
if committed {
|
||||
t.Error("expected false when no changes pending")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("commits accumulated changes with summary", func(t *testing.T) {
|
||||
headBefore, err := store.GetCurrentCommit(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to get HEAD: %v", err)
|
||||
}
|
||||
|
||||
// Create an issue (DML without Dolt commit)
|
||||
issue := &types.Issue{
|
||||
Title: "Batch test issue",
|
||||
Status: types.StatusOpen,
|
||||
Priority: 2,
|
||||
IssueType: types.TypeTask,
|
||||
}
|
||||
if err := store.CreateIssue(ctx, issue, "test-actor"); err != nil {
|
||||
t.Fatalf("CreateIssue failed: %v", err)
|
||||
}
|
||||
|
||||
// Now commit pending changes
|
||||
committed, err := store.CommitPending(ctx, "test-actor")
|
||||
if err != nil {
|
||||
t.Fatalf("CommitPending failed: %v", err)
|
||||
}
|
||||
if !committed {
|
||||
t.Error("expected true when changes were pending")
|
||||
}
|
||||
|
||||
headAfter, err := store.GetCurrentCommit(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to get HEAD after commit: %v", err)
|
||||
}
|
||||
if headAfter == headBefore {
|
||||
t.Error("expected HEAD to advance after CommitPending")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("generates descriptive message", func(t *testing.T) {
|
||||
// Create another issue to have pending changes
|
||||
issue := &types.Issue{
|
||||
Title: "Message test issue",
|
||||
Status: types.StatusOpen,
|
||||
Priority: 2,
|
||||
IssueType: types.TypeTask,
|
||||
}
|
||||
if err := store.CreateIssue(ctx, issue, "test-actor"); err != nil {
|
||||
t.Fatalf("CreateIssue failed: %v", err)
|
||||
}
|
||||
|
||||
// Build the message (without committing)
|
||||
msg := store.buildBatchCommitMessage(ctx, "test-actor")
|
||||
if !strings.Contains(msg, "batch commit") {
|
||||
t.Errorf("expected 'batch commit' in message, got: %q", msg)
|
||||
}
|
||||
if !strings.Contains(msg, "test-actor") {
|
||||
t.Errorf("expected actor in message, got: %q", msg)
|
||||
}
|
||||
if !strings.Contains(msg, "created") {
|
||||
t.Errorf("expected 'created' in message for new issues, got: %q", msg)
|
||||
}
|
||||
|
||||
// Clean up — commit to clear working set
|
||||
if err := store.Commit(ctx, "cleanup"); err != nil {
|
||||
t.Fatalf("cleanup commit failed: %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue