diff --git a/CHANGELOG.md b/CHANGELOG.md index 6988199e1..22a59939b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -736,6 +736,30 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 or without an id) and `bd mol progress` no longer answer empty or refuse over http, and `bd ready --mol` step rows carry their assignee and timestamps. +- **Server-mode config writes are committed again.** On the direct SQL-server + route, `bd remember`, `bd forget` and `bd config set|unset|set-many` wrote + the `config` table but never created a Dolt commit: `maybeAutoCommit` + returns early off the embedded route, and plain `Commit()` excludes config + (GH#2455). The rows sat in the working set, where `bd dolt push` does not + carry them, and a dirty internal key such as `status.custom` made the next + `bd dolt pull` refuse until an explicit `bd dolt commit`. With + `dolt.auto-commit` on (the default) these writes now commit right away + through the scoped `CommitConfigOnly`, which stages only `config` and the + lookup tables projected from it, so a concurrent operation's other dirty + tables are never swept; `batch` and `off` defer them to `bd dolt commit` + like every other write. `bd remember` and `bd forget` commit only user + `kv.*` rows. When an internal config key is dirty (such as the + `issue_prefix` that `bd rename-prefix` writes), the memory is still + stored but not committed: the command warns on stderr with the keys named + and exits 0, `--json` adds `committed: false`, `commit_error` and + `dirty_config_keys`, and `bd dolt commit` publishes the memory with those + keys. v1.0.1 ([#3052](https://github.com/gastownhall/beads/pull/3052)), + whose commits this restores, also warned on a failed memory commit + instead of failing the write. `bd config unset` of a yaml-only secret key + also commits its delete of a credential row an older bd stored in the + database, so HEAD, which `bd dolt push` publishes, no longer keeps it + ([#4078](https://github.com/gastownhall/beads/issues/4078)). + - **PRs based on `hotfix/**` branches now run full CI, not just cross-version historical smokes and triage labeling.** `pr.yml`, `pr-risk.yml`, `conformance.yml`, `cross-version-smoke.yml` and diff --git a/cmd/bd/BUILD.bazel b/cmd/bd/BUILD.bazel index 13895fd37..ca00a91a6 100644 --- a/cmd/bd/BUILD.bazel +++ b/cmd/bd/BUILD.bazel @@ -613,6 +613,8 @@ go_test( "compact_tier2_embedded_test.go", "completions_test.go", "config_apply_test.go", + "config_commit_server_integration_test.go", + "config_commit_write_test.go", "config_drift_test.go", "config_drift_windows_test.go", "config_embedded_test.go", diff --git a/cmd/bd/config.go b/cmd/bd/config.go index 0b4b314bf..6ea9df1fb 100644 --- a/cmd/bd/config.go +++ b/cmd/bd/config.go @@ -20,6 +20,7 @@ import ( "github.com/steveyegge/beads/internal/gitenv" "github.com/steveyegge/beads/internal/metrics" "github.com/steveyegge/beads/internal/remotecache" + "github.com/steveyegge/beads/internal/storage" "github.com/steveyegge/beads/internal/tracker" "github.com/steveyegge/beads/internal/types" "github.com/steveyegge/beads/issueops" @@ -284,6 +285,9 @@ var configSetCmd = &cobra.Command{ return HandleError("setting config: %v", err) } noteDirectConfigWrite() + if err := commitConfigWrite(rootCtx, store, "config set"); err != nil { + return HandleError("%v", err) + } if jsonOutput { if err := outputJSON(map[string]string{ @@ -327,6 +331,22 @@ type secretRowCleanup struct { present bool // a row was there when we read it removed bool err error + // commitErr is a failed Dolt commit of a delete that did land. It is a + // fourth thing to tell the user, not a failed removal: the row is gone from + // the working set, but HEAD, which is what `bd dolt push` publishes, still + // carries it until something commits the delete (GH#4078). After + // storage.ErrCommitIndeterminate the commit may have landed after all, so + // the report says the removal may not be committed rather than that it is + // not (see commitIndeterminate). + commitErr error +} + +// commitIndeterminate reports whether commitErr leaves the delete's commit +// unknown rather than failed: the store marks an error whose outcome may be +// unknown, such as a connection lost during DOLT_COMMIT, with +// storage.ErrCommitIndeterminate. `bd dolt commit` is the remedy either way. +func (c secretRowCleanup) commitIndeterminate() bool { + return errors.Is(c.commitErr, storage.ErrCommitIndeterminate) } // describe writes the database half of the report. It says nothing at all for @@ -337,6 +357,12 @@ func (c secretRowCleanup) describe(key, location string) { case c.removed: fmt.Printf("Also removed a stored %s row from the database, left by a bd that predates the key moving to %s.\n", key, location) fmt.Printf("⚠ Rotate this credential: the row was replicated to every Dolt remote this workspace pushed to, and deleting it here does not unpublish it.\n") + switch { + case c.commitIndeterminate(): + fmt.Fprintf(os.Stderr, "⚠ The removal may not be committed (%v): if it is not, HEAD still carries the row and `bd dolt push` keeps publishing it. Run `bd dolt commit` to be sure.\n", c.commitErr) + case c.commitErr != nil: + fmt.Fprintf(os.Stderr, "⚠ The removal is not committed (%v): HEAD still carries the row and `bd dolt push` keeps publishing it until `bd dolt commit` records the delete.\n", c.commitErr) + } case c.err != nil && c.present: // The one case that is not a maybe: the read found the row and the // delete failed, so bd knows a live credential is still in the table. @@ -350,8 +376,18 @@ func (c secretRowCleanup) describe(key, location string) { // jsonFields carries the same three-way distinction to machine consumers. // database_row_removed is always present for a secret key so a script can tell // "nothing leaked" from "the field is missing because bd is older". +// database_commit_indeterminate qualifies database_commit_failed rather than +// replacing it, so a script that checks only database_commit_failed never takes +// a delete that may be uncommitted, and a credential HEAD may still publish, +// for a committed one. func (c secretRowCleanup) jsonFields(payload map[string]interface{}) { payload["database_row_removed"] = c.removed + if c.commitErr != nil { + payload["database_commit_failed"] = c.commitErr.Error() + if c.commitIndeterminate() { + payload["database_commit_indeterminate"] = true + } + } switch { case c.err != nil && c.present: payload["database_row_present"] = true @@ -366,6 +402,10 @@ func (c secretRowCleanup) jsonFields(payload map[string]interface{}) { // goes on stderr with that error rather than to stdout, which `--json` owns. func (c secretRowCleanup) rowNote(key string) string { switch { + case c.removed && c.commitIndeterminate(): + return fmt.Sprintf("\nA stored %s row WAS removed from the database, but the removal may not be committed (%v); run `bd dolt commit` to be sure, and rotate that credential — it was replicated to every Dolt remote this workspace pushed to.", key, c.commitErr) + case c.removed && c.commitErr != nil: + return fmt.Sprintf("\nA stored %s row WAS removed from the database, but the removal is not committed (%v); run `bd dolt commit`, and rotate that credential — it was replicated to every Dolt remote this workspace pushed to.", key, c.commitErr) case c.removed: return fmt.Sprintf("\nA stored %s row WAS removed from the database; rotate that credential — it was replicated to every Dolt remote this workspace pushed to.", key) case c.err != nil && c.present: @@ -462,10 +502,16 @@ func unsetLeakedSecretRow(key string) secretRowCleanup { return secretRowCleanup{err: err} } present, removed, err := removeStoredSecretRow(rootCtx, settings, key) + cleanup := secretRowCleanup{present: present, removed: removed, err: err} if removed { noteDirectConfigWrite() + // Commit the delete like any other config write (GH#4078): until it is + // committed, HEAD still holds the credential and every `bd dolt push` + // keeps publishing it. A failure is reported, not returned — the delete + // itself landed, and the YAML unset is the command's contract. + cleanup.commitErr = commitConfigWrite(rootCtx, store, "config unset") } - return secretRowCleanup{present: present, removed: removed, err: err} + return cleanup } // removeStoredSecretRow deletes key's row, reporting whether one was there and @@ -910,6 +956,9 @@ var configUnsetCmd = &cobra.Command{ return HandleError("deleting config: %v", err) } noteDirectConfigWrite() + if err := commitConfigWrite(rootCtx, store, "config unset"); err != nil { + return HandleError("%v", err) + } // Clear the config.yaml layer unconditionally and report what the // write actually did. Pre-checking with GetYamlConfig would read @@ -1247,6 +1296,10 @@ Examples: } } commandDidWrite.Store(true) + // One scoped commit for the whole batch, not one per key. + if err := commitConfigWrite(rootCtx, store, "config set-many"); err != nil { + return HandleError("%v", err) + } } } diff --git a/cmd/bd/config_commit_server_integration_test.go b/cmd/bd/config_commit_server_integration_test.go new file mode 100644 index 000000000..72855bd79 --- /dev/null +++ b/cmd/bd/config_commit_server_integration_test.go @@ -0,0 +1,138 @@ +//go:build cgo && integration + +package main + +import ( + "encoding/json" + "fmt" + "os" + "slices" + "strings" + "testing" +) + +// TestCLI_ServerModeConfigWritesCommit_E2E is the user-level pin for GH#4078: +// on the direct SQL-server route, every config-table write verb must leave no +// config dirt in the working set. Before the fix each of them committed the SQL +// transaction only, so the rows (and the custom_statuses / custom_types lookup +// tables SetConfig projects from status.custom and types.custom) sat +// uncommitted: never pushed, and a dirty internal key refused the next pull. +// +// The secret-key step pins the yaml-only branch of `bd config unset`: the +// leaked credential row it deletes must be committed too, or HEAD (what +// `bd dolt push` publishes) keeps it. +// +// The last steps pin the kv.* screen on memory writes: a dirty internal config +// key that the memory write did not make (here the issue_prefix of an unwired +// `bd rename-prefix`) is named and left alone, and the memory command still +// succeeds with a warning, because its own write has landed. +func TestCLI_ServerModeConfigWritesCommit_E2E(t *testing.T) { + skipIfNoDolt(t) + + tmpDir := t.TempDir() + env := os.Environ() + database := uniqueTestDBName(t) + t.Cleanup(func() { + dropTestDatabase(database, testDoltServerPort) + }) + + bd := func(args ...string) (string, error) { + return runBDExecAllowErrorWithEnv(t, tmpDir, env, args...) + } + mustBD := func(args ...string) string { + t.Helper() + out, err := bd(args...) + if err != nil { + t.Fatalf("bd %v failed: %v\n%s", args, err, out) + } + return out + } + // dirtyConfigTables reports which of config and its lookup tables are dirty, + // comma-joined in name order ("" when all are clean). + dirtyConfigTables := func() string { + t.Helper() + out := mustBD("sql", "--csv", + "SELECT table_name FROM dolt_status WHERE table_name IN ('config', 'custom_statuses', 'custom_types') ORDER BY table_name") + var dirty []string + for _, line := range strings.Split(out, "\n") { + switch name := strings.TrimSpace(line); name { + case "config", "custom_statuses", "custom_types": + dirty = append(dirty, name) + } + } + return strings.Join(dirty, ",") + } + + mustBD("init", "--backend", "dolt", "--server", "--external", + "--server-host", "127.0.0.1", + "--server-port", fmt.Sprintf("%d", testDoltServerPort), + "--database", database, + "--prefix", "cfgc", "--quiet") + if got := dirtyConfigTables(); got != "" { + t.Fatalf("precondition: %s dirty right after bd init", got) + } + + for _, step := range [][]string{ + {"remember", "--key", "gh4078", "server-mode memory"}, + {"config", "set", "status.custom", "gh4078review:active"}, + {"config", "set-many", "types.custom=gh4078kind", "status.custom=gh4078review:active,gh4078parked:frozen"}, + {"config", "unset", "types.custom"}, + {"forget", "gh4078"}, + } { + mustBD(step...) + if got := dirtyConfigTables(); got != "" { + t.Fatalf("after bd %s: %s left uncommitted in the working set (GH#4078)", strings.Join(step, " "), got) + } + } + + // A bd that predates notion.token moving to config.yaml stored it in the + // config table, and that row was committed and pushed. + mustBD("sql", "INSERT INTO config (`key`, value) VALUES ('notion.token', 'gh4078-leaked-credential')") + mustBD("dolt", "commit", "-m", "gh4078: an older bd stored notion.token in the database") + if got := dirtyConfigTables(); got != "" { + t.Fatalf("precondition: %s dirty after bd dolt commit", got) + } + if out := mustBD("config", "unset", "notion.token"); !strings.Contains(out, "Also removed a stored notion.token row") { + t.Fatalf("bd config unset notion.token did not remove the stored row:\n%s", out) + } + if got := dirtyConfigTables(); got != "" { + t.Fatalf("after bd config unset notion.token: %s left uncommitted, so HEAD still publishes the credential (GH#4078)", got) + } + + mustBD("rename-prefix", "rvx-") + if got := dirtyConfigTables(); got != "config" { + t.Fatalf("precondition: dirty = %q after bd rename-prefix, want its unwired issue_prefix write in config", got) + } + stdout, stderr, err := runBDExecSeparated(t, testBD, tmpDir, env, "remember", "--key", "gh4078-screened", "screened memory", "--json") + if err != nil { + t.Fatalf("bd remember failed after its write landed: %v\nstdout:\n%s\nstderr:\n%s", err, stdout, stderr) + } + var remembered struct { + Key string `json:"key"` + Committed *bool `json:"committed"` + DirtyConfigKeys []string `json:"dirty_config_keys"` + } + if err := json.Unmarshal([]byte(stdout), &remembered); err != nil { + t.Fatalf("bd remember --json stdout is not one JSON object: %v\n%s", err, stdout) + } + if remembered.Key != "gh4078-screened" || remembered.Committed == nil || *remembered.Committed || + !slices.Equal(remembered.DirtyConfigKeys, []string{"issue_prefix"}) { + t.Fatalf("bd remember --json = %s, want key gh4078-screened, committed false, dirty_config_keys [issue_prefix]", stdout) + } + if !strings.Contains(stderr, "Warning:") || !strings.Contains(stderr, "issue_prefix") { + t.Fatalf("bd remember stderr does not warn about the refused commit naming issue_prefix:\n%s", stderr) + } + if out := mustBD("recall", "gh4078-screened"); !strings.Contains(out, "screened memory") { + t.Fatalf("the screened memory was not stored:\n%s", out) + } + if out := mustBD("forget", "gh4078-screened"); !strings.Contains(out, "Warning:") || !strings.Contains(out, "issue_prefix") { + t.Fatalf("bd forget did not warn about the refused commit naming issue_prefix:\n%s", out) + } + if got := dirtyConfigTables(); got != "config" { + t.Fatalf("after the refused memory commits: dirty = %q, want config left for bd dolt commit", got) + } + mustBD("dolt", "commit", "-m", "gh4078: commit the screened config") + if got := dirtyConfigTables(); got != "" { + t.Fatalf("after bd dolt commit: %s still dirty", got) + } +} diff --git a/cmd/bd/config_commit_write_test.go b/cmd/bd/config_commit_write_test.go new file mode 100644 index 000000000..e14daa745 --- /dev/null +++ b/cmd/bd/config_commit_write_test.go @@ -0,0 +1,490 @@ +//go:build cgo + +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "reflect" + "slices" + "strings" + "testing" + + "github.com/steveyegge/beads/internal/storage" + "github.com/steveyegge/beads/memoryops" +) + +type fakeConfigCommitStore struct { + storage.DoltStorage + configOnlyCalls int + userKVCalls int + messages []string + err error +} + +func (f *fakeConfigCommitStore) CommitConfigOnly(_ context.Context, message string) error { + f.configOnlyCalls++ + f.messages = append(f.messages, message) + return f.err +} + +func (f *fakeConfigCommitStore) CommitConfigUserKVOnly(_ context.Context, message string) error { + f.userKVCalls++ + f.messages = append(f.messages, message) + return f.err +} + +// GH#4078: server-mode config writes (bd remember, bd config set) must be +// committed immediately and scoped — maybeAutoCommit skips server mode +// entirely and generic Commit() excludes config, so nothing else ever +// commits them. +func TestCommitConfigWriteServerModeCommitsScoped(t *testing.T) { + saveStorageMode(t) + serverMode = true + + fake := &fakeConfigCommitStore{} + if err := commitConfigWrite(context.Background(), fake, "remember"); err != nil { + t.Fatalf("commitConfigWrite: %v", err) + } + if fake.configOnlyCalls != 1 { + t.Fatalf("CommitConfigOnly calls = %d, want 1 in server mode", fake.configOnlyCalls) + } + if fake.userKVCalls != 0 { + t.Fatalf("CommitConfigUserKVOnly calls = %d, want 0 for a config write", fake.userKVCalls) + } + if !strings.HasPrefix(fake.messages[0], "bd: remember (auto-commit) by ") { + t.Fatalf("commit message = %q, want prefix %q", fake.messages[0], "bd: remember (auto-commit) by ") + } +} + +// A memory write commits through the kv.*-screened variant, so a concurrent +// writer's dirty internal config key is refused instead of swept in. +func TestCommitMemoryWriteServerModeCommitsUserKVScreened(t *testing.T) { + saveStorageMode(t) + serverMode = true + + fake := &fakeConfigCommitStore{} + if err := commitMemoryWrite(context.Background(), fake, "remember"); err != nil { + t.Fatalf("commitMemoryWrite: %v", err) + } + if fake.userKVCalls != 1 { + t.Fatalf("CommitConfigUserKVOnly calls = %d, want 1 in server mode", fake.userKVCalls) + } + if fake.configOnlyCalls != 0 { + t.Fatalf("CommitConfigOnly calls = %d, want 0 for a memory write", fake.configOnlyCalls) + } +} + +// The screen's refusal reaches the user rather than being taken for a +// nothing-to-commit no-op, and keeps its type through the wrap: the --json +// payload names the refused keys from it. +func TestCommitMemoryWriteSurfacesScreenRefusal(t *testing.T) { + saveStorageMode(t) + serverMode = true + + fake := &fakeConfigCommitStore{err: &storage.ConfigCommitRefusedError{Keys: []string{"issue_prefix"}}} + err := commitMemoryWrite(context.Background(), fake, "forget") + var refused *storage.ConfigCommitRefusedError + if !errors.As(err, &refused) || !slices.Equal(refused.Keys, []string{"issue_prefix"}) { + t.Fatalf("commitMemoryWrite error = %v, want the screen's refusal naming issue_prefix", err) + } +} + +// memoryCommitFields is how --json learns that a memory write landed but was +// not committed, which the exit code no longer says. A committed write adds +// nothing, so its payload keeps the shipped shape. A commit that may have +// landed is qualified, not reported as committed: committed stays false. +func TestMemoryCommitFields(t *testing.T) { + refusal := fmt.Errorf("committing memory write: %w", + &storage.ConfigCommitRefusedError{Keys: []string{"issue_prefix", "sync.branch"}}) + failure := errors.New("committing memory write: connection refused") + indeterminate := fmt.Errorf("committing memory write: %w", errIndeterminateCommit) + for _, tc := range []struct { + name string + commitErr error + want map[string]interface{} + }{ + {name: "committed", want: map[string]interface{}{"key": "k"}}, + {name: "refused by the kv.* screen", commitErr: refusal, want: map[string]interface{}{ + "key": "k", + "committed": false, + "commit_error": refusal.Error(), + "dirty_config_keys": []string{"issue_prefix", "sync.branch"}, + }}, + {name: "failed for another reason", commitErr: failure, want: map[string]interface{}{ + "key": "k", + "committed": false, + "commit_error": failure.Error(), + }}, + {name: "may have landed", commitErr: indeterminate, want: map[string]interface{}{ + "key": "k", + "committed": false, + "commit_error": indeterminate.Error(), + "commit_indeterminate": true, + }}, + } { + t.Run(tc.name, func(t *testing.T) { + payload := map[string]interface{}{"key": "k"} + memoryCommitFields(payload, tc.commitErr) + if !reflect.DeepEqual(payload, tc.want) { + t.Fatalf("payload = %#v, want %#v", payload, tc.want) + } + }) + } +} + +// errIndeterminateCommit is what the SQL-server store returns when the +// connection drops during DOLT_COMMIT: the commit may or may not have landed. +var errIndeterminateCommit = fmt.Errorf("dolt commit after SQL mutation: %w: %w", + errors.New("invalid connection"), storage.ErrCommitIndeterminate) + +// refusingMemoryStore is a SQL-server store on which a memory write lands and +// its kv.*-screened commit is refused: what a dirty internal key, such as the +// issue_prefix `bd rename-prefix` leaves, makes of every remember and forget. +type refusingMemoryStore struct { + storage.DoltStorage + memories *landingMemories + commitErr error + commits int +} + +func (s *refusingMemoryStore) Memories() (memoryops.Memories, error) { return s.memories, nil } + +func (s *refusingMemoryStore) CommitConfigUserKVOnly(context.Context, string) error { + s.commits++ + return s.commitErr +} + +type landingMemories struct { + memoryops.Memories + stored map[string]string +} + +func (m *landingMemories) Remember(_ context.Context, req memoryops.RememberRequest) (memoryops.RememberResult, error) { + _, replaced := m.stored[req.Key] + m.stored[req.Key] = req.Content + return memoryops.RememberResult{Key: req.Key, Value: req.Content, Replaced: replaced}, nil +} + +func (m *landingMemories) Forget(_ context.Context, req memoryops.ForgetRequest) (memoryops.ForgetResult, error) { + value, found := m.stored[req.Key] + delete(m.stored, req.Key) + return memoryops.ForgetResult{Key: req.Key, Value: value, Found: found}, nil +} + +// A refused memory commit must not turn a write that landed into exit 1: +// remember and forget warn on stderr, print their normal result and, under +// --json, add committed:false and the refused keys. v1.0.1 (#3052) warned and +// carried on the same way. +func TestMemoryCommandsWarnAndSucceedWhenTheCommitIsRefused(t *testing.T) { + for _, asJSON := range []bool{false, true} { + t.Run(fmt.Sprintf("json=%v", asJSON), func(t *testing.T) { + t.Setenv("BEADS_DIR", t.TempDir()) + saveAndRestoreGlobals(t) + saveStorageMode(t) + serverMode = true + fake := &refusingMemoryStore{ + memories: &landingMemories{stored: map[string]string{}}, + commitErr: &storage.ConfigCommitRefusedError{Keys: []string{"issue_prefix"}}, + } + store = fake + setStoreActive(true) + savedKey, savedJSON, savedConfigInt := memoryKeyFlag, jsonOutput, memoryConfigInt + t.Cleanup(func() { memoryKeyFlag, jsonOutput, memoryConfigInt = savedKey, savedJSON, savedConfigInt }) + memoryKeyFlag, jsonOutput = "gh4078", asJSON + memoryConfigInt = func(string) int { return 0 } + + for _, step := range []struct { + cmd string + run func() error + text string + json map[string]interface{} + }{ + { + cmd: "remember", + run: func() error { return rememberCmd.RunE(rememberCmd, []string{"a memory that lands"}) }, + text: "Remembered [gh4078]: a memory that lands", + json: map[string]interface{}{"key": "gh4078", "value": "a memory that lands", "action": "remembered"}, + }, + { + cmd: "forget", + run: func() error { return forgetCmd.RunE(forgetCmd, []string{"gh4078"}) }, + text: "Forgot [gh4078]: a memory that lands", + json: map[string]interface{}{"key": "gh4078", "deleted": "true"}, + }, + } { + var runErr error + stdout, stderr := captureStdio(t, func() { runErr = step.run() }) + if runErr != nil { + t.Fatalf("bd %s returned %v after its write landed, want success with a warning", step.cmd, runErr) + } + if !strings.Contains(stderr, "Warning: committing memory write: refusing to commit") || + !strings.Contains(stderr, "issue_prefix") { + t.Errorf("bd %s stderr does not warn about the refused commit naming issue_prefix:\n%s", step.cmd, stderr) + } + if !asJSON { + if !strings.Contains(stdout, step.text) { + t.Errorf("bd %s stdout = %q, want the normal result %q", step.cmd, stdout, step.text) + } + continue + } + var got map[string]interface{} + if err := json.Unmarshal([]byte(stdout), &got); err != nil { + t.Fatalf("bd %s --json stdout is not one JSON object: %v\n%s", step.cmd, err, stdout) + } + want := step.json + want["schema_version"] = float64(JSONSchemaVersion) + want["committed"] = false + want["commit_error"] = "committing memory write: " + fake.commitErr.Error() + want["dirty_config_keys"] = []interface{}{"issue_prefix"} + if !reflect.DeepEqual(got, want) { + t.Errorf("bd %s --json = %#v, want %#v", step.cmd, got, want) + } + } + if fake.commits != 2 { + t.Errorf("CommitConfigUserKVOnly calls = %d, want one per command", fake.commits) + } + if len(fake.memories.stored) != 0 { + t.Errorf("stored memories = %v, want the forget to have landed too", fake.memories.stored) + } + }) + } +} + +// The proxied route's role commits each config write inside its own unit of +// work, so committing again here would be a second commit for the same write. +func TestCommitConfigWriteProxiedServerModeIsNoOp(t *testing.T) { + saveStorageMode(t) + proxiedServerMode = true + + fake := &fakeConfigCommitStore{} + if err := commitConfigWrite(context.Background(), fake, "config set"); err != nil { + t.Fatalf("commitConfigWrite: %v", err) + } + if fake.configOnlyCalls != 0 { + t.Fatalf("CommitConfigOnly calls = %d, want 0 in proxied server mode", fake.configOnlyCalls) + } +} + +// batch and off defer server-mode version commits to `bd dolt commit`, whose +// CommitAll includes config, so a config write must defer with them. +func TestCommitConfigWriteHonorsBatchAndOffModes(t *testing.T) { + for _, mode := range []doltAutoCommitMode{doltAutoCommitBatch, doltAutoCommitOff} { + t.Run(string(mode), func(t *testing.T) { + saveStorageMode(t) + serverMode = true + doltAutoCommit = string(mode) + + fake := &fakeConfigCommitStore{} + if err := commitConfigWrite(context.Background(), fake, "remember"); err != nil { + t.Fatalf("commitConfigWrite: %v", err) + } + if fake.configOnlyCalls != 0 { + t.Fatalf("CommitConfigOnly calls = %d, want 0 in %s mode", fake.configOnlyCalls, mode) + } + }) + } +} + +// Embedded mode already commits config through the PersistentPostRun +// auto-commit, so the helper must not double-commit there. +func TestCommitConfigWriteEmbeddedModeIsNoOp(t *testing.T) { + saveStorageMode(t) + serverMode = false + proxiedServerMode = false + + fake := &fakeConfigCommitStore{} + if err := commitConfigWrite(context.Background(), fake, "remember"); err != nil { + t.Fatalf("commitConfigWrite: %v", err) + } + if fake.configOnlyCalls != 0 { + t.Fatalf("CommitConfigOnly calls = %d, want 0 in embedded mode", fake.configOnlyCalls) + } +} + +// A real commit failure must surface — silently losing the commit is the +// exact bug class this helper exists to fix (GH#4078's silent no-op half). +func TestCommitConfigWriteSurfacesCommitError(t *testing.T) { + saveStorageMode(t) + serverMode = true + + fake := &fakeConfigCommitStore{err: errors.New("connection refused")} + err := commitConfigWrite(context.Background(), fake, "remember") + if err == nil { + t.Fatal("commitConfigWrite returned nil, want the commit error surfaced") + } + if !strings.Contains(err.Error(), "connection refused") { + t.Fatalf("error = %q, want it to wrap %q", err.Error(), "connection refused") + } +} + +// "Nothing to commit" is a benign race (e.g. the same key re-written to an +// identical value) and must not fail the user's write. +func TestCommitConfigWriteSwallowsNothingToCommit(t *testing.T) { + saveStorageMode(t) + serverMode = true + + fake := &fakeConfigCommitStore{err: errors.New("nothing to commit")} + if err := commitConfigWrite(context.Background(), fake, "remember"); err != nil { + t.Fatalf("commitConfigWrite: %v, want nothing-to-commit swallowed", err) + } + if fake.configOnlyCalls != 1 { + t.Fatalf("CommitConfigOnly calls = %d, want 1", fake.configOnlyCalls) + } +} + +// The yaml-only secret branch of `bd config unset` deletes a credential row an +// older bd left in the config table. On the SQL-server route only this commit +// takes the delete to HEAD, the commit `bd dolt push` publishes, so it must +// happen; and since the delete has landed by then, a failed commit warns +// rather than failing the command. +func TestConfigUnsetCommitsTheLeakedRowDelete(t *testing.T) { + setup := func(t *testing.T, stored map[string]string, commitErr error) *leakedRowStore { + t.Helper() + newLeakedSecretWorkspace(t, "notion:\n token: yaml-copy\n", true) + installLeakedRowSettings(t, &recordingWorkspaceConfig{stored: stored}) + saveStorageMode(t) + serverMode = true + fake := store.(*leakedRowStore) + fake.commitErr = commitErr + return fake + } + + t.Run("a removed row is committed", func(t *testing.T) { + fake := setup(t, map[string]string{"notion.token": "leaked-db-value"}, nil) + stdout, stderr, err := runConfigUnset(t, "notion.token", false) + if err != nil { + t.Fatalf("config unset notion.token: %v", err) + } + if len(fake.commits) != 1 || !strings.HasPrefix(fake.commits[0], "bd: config unset (auto-commit) by ") { + t.Fatalf("commits = %q, want one config unset auto-commit — HEAD still publishes the credential", fake.commits) + } + if !strings.Contains(stdout, "Also removed a stored notion.token row") { + t.Errorf("the removal was not reported; stdout:\n%s", stdout) + } + if strings.Contains(stderr, "not committed") { + t.Errorf("a committed removal was reported as uncommitted; stderr:\n%s", stderr) + } + }) + + t.Run("no row: nothing to commit", func(t *testing.T) { + fake := setup(t, map[string]string{}, nil) + if _, _, err := runConfigUnset(t, "notion.token", false); err != nil { + t.Fatalf("config unset notion.token: %v", err) + } + if len(fake.commits) != 0 { + t.Errorf("commits = %q, want none when no row was removed", fake.commits) + } + }) + + t.Run("a failed commit warns and the unset still succeeds", func(t *testing.T) { + fake := setup(t, map[string]string{"notion.token": "leaked-db-value"}, errors.New("connection refused")) + stdout, stderr, err := runConfigUnset(t, "notion.token", false) + if err != nil { + t.Fatalf("config unset notion.token returned %v, want success: the delete and the YAML unset both landed", err) + } + if len(fake.commits) != 1 { + t.Fatalf("commits = %q, want the one attempt", fake.commits) + } + if !strings.Contains(stdout, "Also removed a stored notion.token row") { + t.Errorf("the removal was not reported; stdout:\n%s", stdout) + } + for _, want := range []string{"The removal is not committed", "connection refused", "bd dolt commit"} { + if !strings.Contains(stderr, want) { + t.Errorf("stderr does not carry %q:\n%s", want, stderr) + } + } + }) + + t.Run("--json reports a failed commit", func(t *testing.T) { + setup(t, map[string]string{"notion.token": "leaked-db-value"}, errors.New("connection refused")) + stdout, _, err := runConfigUnset(t, "notion.token", true) + if err != nil { + t.Fatalf("config unset notion.token --json: %v", err) + } + var got map[string]interface{} + if err := json.Unmarshal([]byte(stdout), &got); err != nil { + t.Fatalf("stdout is not one JSON object: %v\n%s", err, stdout) + } + if got["database_row_removed"] != true { + t.Errorf("database_row_removed = %v, want true", got["database_row_removed"]) + } + if msg, _ := got["database_commit_failed"].(string); !strings.Contains(msg, "connection refused") { + t.Errorf("database_commit_failed = %v, want the commit error", got["database_commit_failed"]) + } + if v, ok := got["database_commit_indeterminate"]; ok { + t.Errorf("database_commit_indeterminate = %v for a commit that failed outright, want it absent", v) + } + }) + + // After storage.ErrCommitIndeterminate the commit may have landed, so the + // report must not assert that HEAD still carries the row; and --json keeps + // database_commit_failed beside the qualifier, so a script that reads only + // that field is still told to commit. + t.Run("an indeterminate commit is hedged, not reported as failed", func(t *testing.T) { + setup(t, map[string]string{"notion.token": "leaked-db-value"}, errIndeterminateCommit) + _, stderr, err := runConfigUnset(t, "notion.token", false) + if err != nil { + t.Fatalf("config unset notion.token returned %v, want success: the delete and the YAML unset both landed", err) + } + for _, want := range []string{"The removal may not be committed", "write commit result indeterminate", "bd dolt commit"} { + if !strings.Contains(stderr, want) { + t.Errorf("stderr does not carry %q:\n%s", want, stderr) + } + } + if strings.Contains(stderr, "The removal is not committed") { + t.Errorf("a commit that may have landed was reported as not committed; stderr:\n%s", stderr) + } + }) + + t.Run("--json qualifies an indeterminate commit", func(t *testing.T) { + setup(t, map[string]string{"notion.token": "leaked-db-value"}, errIndeterminateCommit) + stdout, _, err := runConfigUnset(t, "notion.token", true) + if err != nil { + t.Fatalf("config unset notion.token --json: %v", err) + } + var got map[string]interface{} + if err := json.Unmarshal([]byte(stdout), &got); err != nil { + t.Fatalf("stdout is not one JSON object: %v\n%s", err, stdout) + } + if msg, _ := got["database_commit_failed"].(string); !strings.Contains(msg, "write commit result indeterminate") { + t.Errorf("database_commit_failed = %v, want the commit error kept beside the qualifier", got["database_commit_failed"]) + } + if got["database_commit_indeterminate"] != true { + t.Errorf("database_commit_indeterminate = %v, want true", got["database_commit_indeterminate"]) + } + }) +} + +// rowNote folds the database half into the error a failed YAML unset returns, +// so it hedges an indeterminate commit the same way describe does. +func TestSecretRowCleanupRowNoteHedgesAnIndeterminateCommit(t *testing.T) { + for _, tc := range []struct { + name string + commitErr error + want, notWant string + }{ + {name: "failed", commitErr: errors.New("connection refused"), + want: "the removal is not committed", notWant: "may not be committed"}, + {name: "indeterminate", commitErr: errIndeterminateCommit, + want: "the removal may not be committed", notWant: "is not committed"}, + } { + t.Run(tc.name, func(t *testing.T) { + note := secretRowCleanup{present: true, removed: true, commitErr: tc.commitErr}.rowNote("notion.token") + if !strings.Contains(note, tc.want) || strings.Contains(note, tc.notWant) { + t.Errorf("rowNote = %q, want %q and not %q", note, tc.want, tc.notWant) + } + }) + } +} + +func TestCommitConfigWriteNilStoreIsNoOp(t *testing.T) { + saveStorageMode(t) + serverMode = true + + if err := commitConfigWrite(context.Background(), nil, "remember"); err != nil { + t.Fatalf("commitConfigWrite with nil store: %v, want no-op", err) + } +} diff --git a/cmd/bd/config_leaked_secret_test.go b/cmd/bd/config_leaked_secret_test.go index fb4418271..3e92c1936 100644 --- a/cmd/bd/config_leaked_secret_test.go +++ b/cmd/bd/config_leaked_secret_test.go @@ -238,18 +238,28 @@ func TestNotionStatusWarnsOnlyForTheLeakedSource(t *testing.T) { } } -// leakedRowStore is the production store reduced to the one accessor the -// cleanup reaches through. Its nil embedded storage makes any other call panic, -// so these tests cannot silently exercise more of the store than the seam. +// leakedRowStore is the production store reduced to the two calls the cleanup +// makes: the config-plane accessor, and the scoped commit of a delete that +// landed (on the SQL-server route only; elsewhere commitConfigWrite is a no-op +// before it reaches the store). Its nil embedded storage makes any other call +// panic, so these tests cannot silently exercise more of the store than that. type leakedRowStore struct { storage.DoltStorage settings *recordingWorkspaceConfig + + commits []string + commitErr error } func (s *leakedRowStore) WorkspaceConfig() (issueops.WorkspaceConfig, error) { return s.settings, nil } +func (s *leakedRowStore) CommitConfigOnly(_ context.Context, message string) error { + s.commits = append(s.commits, message) + return s.commitErr +} + // newLeakedSecretWorkspace builds the workspace shapes GH#6676 produced: a // .beads directory that may or may not hold a config.yaml, and may or may not // hold a database directory — which is what FindDatabasePath answers on, and diff --git a/cmd/bd/dolt_autocommit.go b/cmd/bd/dolt_autocommit.go index a32205e85..96fe62e4c 100644 --- a/cmd/bd/dolt_autocommit.go +++ b/cmd/bd/dolt_autocommit.go @@ -155,6 +155,77 @@ func commitPendingIfEmbedded(ctx context.Context, st storage.DoltStorage, actor return maybeAutoCommitStore(ctx, st, p) } +// commitConfigWrite creates the Dolt commit for an intentional config-table +// write on the direct SQL-server route (bd config set/unset/set-many; bd +// remember/forget go through commitMemoryWrite). Nothing else commits it there +// (GH#4078): the store-level writes those verbs reach through their roles +// commit the SQL transaction only, maybeAutoCommit returns early off the +// embedded route, and plain Commit excludes config anyway (GH#2455). +// CommitConfigOnly stages ONLY the config table, so a concurrent operation's +// other dirty tables are never swept. +// +// It is a no-op wherever something else already owns that commit: embedded +// mode (the PersistentPostRun auto-commit stages config with everything +// else), the proxied route (the role's unit of work commits each write), and +// dolt.auto-commit batch/off, which defer it to `bd dolt commit` (CommitAll, +// config included) exactly as they defer every other server-mode write. +// +// Unlike commitMemoryWrite it does not screen other dirty config rows: the +// operator is writing config on purpose, the trust level of `bd dolt commit`. +// The config verbs return a failed commit as the command's failure, where +// remember and forget only warn: a memory commit is refused whenever another +// writer left an internal key dirty, so exit 1 there would be routine for a +// write that landed, while this commit is never refused and fails only on a +// genuine error, which its "committing config write" prefix attributes to the +// commit rather than the write. (config unset's delete of a leaked secret row +// warns too: the YAML unset is that command's contract.) +// rename-prefix and migrate also write config (issue_prefix, sync.branch) and +// are deliberately not wired here: they are multi-step operations, and when +// their config row should become a commit is the GH#2455 question itself. +// `bd kv set`/`clear` and the kv.memory rows `bd import` writes are not wired +// here yet either, so on this route they stay in the working set until a later +// commit stages config, such as `bd dolt commit`. +func commitConfigWrite(ctx context.Context, st storage.DoltStorage, command string) error { + if err := commitConfigTable(ctx, st, command, storage.DoltStorage.CommitConfigOnly); err != nil { + return fmt.Errorf("committing config write: %w", err) + } + return nil +} + +// commitMemoryWrite is commitConfigWrite for bd remember/forget. A memory is +// user kv.* data, so its commit takes only kv.* config rows with it +// (CommitConfigUserKVOnly): a dirty internal key is refused, not swept in. The +// refusal is a *storage.ConfigCommitRefusedError naming the keys, and the +// callers report it as a warning: by then the memory write has landed. +func commitMemoryWrite(ctx context.Context, st storage.DoltStorage, command string) error { + if err := commitConfigTable(ctx, st, command, storage.DoltStorage.CommitConfigUserKVOnly); err != nil { + return fmt.Errorf("committing memory write: %w", err) + } + return nil +} + +func commitConfigTable(ctx context.Context, st storage.DoltStorage, command string, + commit func(storage.DoltStorage, context.Context, string) error) error { + if isEmbeddedMode() || usesProxiedServer() || st == nil { + return nil + } + commitNow, err := writesCommitNow() + if err != nil { + return err + } + if !commitNow { + return nil + } + if lm, ok := storage.UnwrapStore(st).(storage.LifecycleManager); ok && lm.IsClosed() { + return nil + } + msg := formatDoltAutoCommitMessage(command, getActor(), nil) + if err := commit(st, ctx, msg); err != nil && !isDoltNothingToCommit(err) { + return err + } + return nil +} + func maybeAutoCommitStore(ctx context.Context, st storage.DoltStorage, p doltAutoCommitParams) error { mode, err := getDoltAutoCommitMode() if err != nil { diff --git a/cmd/bd/memory.go b/cmd/bd/memory.go index 1f0746b05..46fce8cd5 100644 --- a/cmd/bd/memory.go +++ b/cmd/bd/memory.go @@ -12,6 +12,7 @@ import ( "github.com/steveyegge/beads/internal/config" "github.com/steveyegge/beads/internal/memoryapi" "github.com/steveyegge/beads/internal/metrics" + "github.com/steveyegge/beads/internal/storage" "github.com/steveyegge/beads/internal/storage/kvkeys" "github.com/steveyegge/beads/internal/storage/uow" "github.com/steveyegge/beads/memoryops" @@ -243,19 +244,47 @@ func rememberBareKeyPath(key, insight, existing string, found bool) error { key, insight, key) } -// printRememberResult renders the `bd remember` success output. -func printRememberResult(verb, key, insight string) error { +// printRememberResult renders the `bd remember` success output. commitErr is +// the memory commit's failure, if any; the memory is stored either way. +func printRememberResult(verb, key, insight string, commitErr error) error { if jsonOutput { - return outputJSON(map[string]string{ + payload := map[string]interface{}{ "key": key, "value": insight, "action": strings.ToLower(verb), - }) + } + memoryCommitFields(payload, commitErr) + return outputJSON(payload) } fmt.Printf("%s [%s]: %s\n", verb, key, truncateMemory(insight, 80)) return nil } +// memoryCommitFields tells a `--json` consumer that the write landed but its +// Dolt commit did not, which the exit code no longer says (see the commit +// sites). The fields are added only then, so a committed write's payload keeps +// its shipped shape. dirty_config_keys names the internal keys the kv.* screen +// refused, which is what the user has to commit or revert. commit_indeterminate +// marks an error after which the store cannot say whether the commit landed +// (storage.ErrCommitIndeterminate, such as a connection lost during +// DOLT_COMMIT). committed stays false beside it, so a consumer that reads only +// committed never takes a memory that may be uncommitted for a committed one; +// `bd dolt commit` settles it either way. +func memoryCommitFields(payload map[string]interface{}, commitErr error) { + if commitErr == nil { + return + } + payload["committed"] = false + payload["commit_error"] = commitErr.Error() + if errors.Is(commitErr, storage.ErrCommitIndeterminate) { + payload["commit_indeterminate"] = true + } + var refused *storage.ConfigCommitRefusedError + if errors.As(commitErr, &refused) { + payload["dirty_config_keys"] = refused.Keys + } +} + // printMemoriesResult renders the `bd memories` output. func printMemoriesResult(memories map[string]string, search string) error { if jsonOutput { @@ -306,13 +335,16 @@ func printForgetNotFound(key string) error { return SilentExit() } -// printForgetResult renders the `bd forget` success output. -func printForgetResult(key, existing string) error { +// printForgetResult renders the `bd forget` success output. commitErr is as +// for printRememberResult. +func printForgetResult(key, existing string, commitErr error) error { if jsonOutput { - return outputJSON(map[string]string{ + payload := map[string]interface{}{ "key": key, "deleted": "true", - }) + } + memoryCommitFields(payload, commitErr) + return outputJSON(payload) } fmt.Printf("Forgot [%s]: %s\n", key, truncateMemory(existing, 80)) return nil @@ -518,6 +550,15 @@ Examples: return HandleErrorRespectJSON("storing memory: %v", err) } noteDirectMemoryWrite() + // The memory is stored by now, so a commit the kv.* screen refuses (a + // dirty internal key such as `bd rename-prefix` leaves) or that fails + // is a warning, not the command's failure: exit 1 would report a write + // that landed as one that did not. v1.0.1 (#3052) warned and carried on + // the same way; --json gets the same news in memoryCommitFields. + commitErr := commitMemoryWrite(rootCtx, store, "remember") + if commitErr != nil { + WarnError("%v", commitErr) + } if budgetLine != "" { fmt.Fprintln(os.Stderr, budgetLine) @@ -530,7 +571,7 @@ Examples: if result.Replaced { verb = "Updated" } - return printRememberResult(verb, result.Key, result.Value) + return printRememberResult(verb, result.Key, result.Value, commitErr) }, } @@ -622,8 +663,14 @@ Examples: return printForgetNotFound(result.Key) } noteDirectMemoryWrite() + // As for remember: the delete landed, so a refused or failed commit + // warns and the command still succeeds. + commitErr := commitMemoryWrite(rootCtx, store, "forget") + if commitErr != nil { + WarnError("%v", commitErr) + } - return printForgetResult(result.Key, result.Value) + return printForgetResult(result.Key, result.Value, commitErr) }, } diff --git a/internal/httpclient/store.go b/internal/httpclient/store.go index 225938942..c03ac6d2d 100644 --- a/internal/httpclient/store.go +++ b/internal/httpclient/store.go @@ -249,3 +249,10 @@ func (s *Store) CommitAll(_ context.Context, _ string) (bool, error) { return fa // CommitPending reports nothing pending, for the same reason: write paths call // it opportunistically and must not error. func (s *Store) CommitPending(_ context.Context, _ string) (bool, error) { return false, nil } + +// CommitConfigOnly and CommitConfigUserKVOnly, the scoped commits after a +// config or memory write (GH#4078), join the family for the same D3 reason: +// the write is durable once the server answers, so there is nothing to flush, +// and refusing would fail a command whose write already landed. +func (s *Store) CommitConfigOnly(_ context.Context, _ string) error { return nil } +func (s *Store) CommitConfigUserKVOnly(_ context.Context, _ string) error { return nil } diff --git a/internal/httpclient/store_test.go b/internal/httpclient/store_test.go index 117c77c4b..fd5c3009c 100644 --- a/internal/httpclient/store_test.go +++ b/internal/httpclient/store_test.go @@ -79,7 +79,8 @@ func pinnedProjectTarget(t *testing.T) Target { // TestCommitFamilyIsNoOp is the D3 posture. Without it `bd ready --claim` exits // 1 after a durable claim: auto-commit defaults on for a registered workspace, // the claim path calls Commit, and PostRun turns any Commit error into a failed -// exit. +// exit. The scoped config commits fail the same way: `bd config set` and `bd +// remember` call them after a write that already landed (GH#4078). func TestCommitFamilyIsNoOp(t *testing.T) { s := New(pinnedProjectTarget(t), nil, nil) ctx := context.Background() @@ -93,6 +94,12 @@ func TestCommitFamilyIsNoOp(t *testing.T) { if err := s.CommitMergeResolution(ctx, "msg"); err != nil { t.Errorf("CommitMergeResolution: %v, want nil", err) } + if err := s.CommitConfigOnly(ctx, "msg"); err != nil { + t.Errorf("CommitConfigOnly: %v, want nil", err) + } + if err := s.CommitConfigUserKVOnly(ctx, "msg"); err != nil { + t.Errorf("CommitConfigUserKVOnly: %v, want nil", err) + } pending, err := s.CommitPending(ctx, "actor") if err != nil { t.Errorf("CommitPending: %v, want nil", err) @@ -116,7 +123,10 @@ func TestCommitFamilyIsNoOp(t *testing.T) { // commit family ever drifts onto the allowlist, the no-ops above are dead code // and the claim path breaks. func TestCommitFamilyIsNotOnTheUnsupportedMap(t *testing.T) { - for _, name := range []string{"Commit", "CommitWithConfig", "CommitMergeResolution", "CommitAll", "CommitPending", "Close"} { + for _, name := range []string{ + "Commit", "CommitWithConfig", "CommitMergeResolution", "CommitAll", "CommitPending", + "CommitConfigOnly", "CommitConfigUserKVOnly", "Close", + } { if reason, ok := legitimatelyUnsupported[name]; ok { t.Errorf("%q is on the unsupported allowlist (%q) but D3/D4 require a no-op", name, reason) } diff --git a/internal/httpclient/unsupported_gen.go b/internal/httpclient/unsupported_gen.go index 7c900ccb7..c92f73d69 100644 --- a/internal/httpclient/unsupported_gen.go +++ b/internal/httpclient/unsupported_gen.go @@ -727,4 +727,4 @@ func (unsupportedDoltStorage) VersionReconciler() (_ issueops.VersionReconciler, return } -// NOTE: partial shell (142 of 201 methods generated; 59 left to this package's hand-written set). +// NOTE: partial shell (142 of 203 methods generated; 61 left to this package's hand-written set). diff --git a/internal/storage/BUILD.bazel b/internal/storage/BUILD.bazel index acb3b8e08..7252f4b68 100644 --- a/internal/storage/BUILD.bazel +++ b/internal/storage/BUILD.bazel @@ -70,6 +70,7 @@ go_library( visibility = ["//:__subpackages__"], deps = [ "//internal/hooks", + "//internal/storage/kvkeys", "//internal/types", "//issueops", "//journalops", diff --git a/internal/storage/dolt/BUILD.bazel b/internal/storage/dolt/BUILD.bazel index 7d4572d16..f81b80beb 100644 --- a/internal/storage/dolt/BUILD.bazel +++ b/internal/storage/dolt/BUILD.bazel @@ -168,6 +168,7 @@ go_test( "commenter_persistence_test.go", "comments_page_test.go", "commit_all_test.go", + "commit_config_test.go", "commit_dirty_wisp_test.go", "compact_test.go", "concurrent_test.go", diff --git a/internal/storage/dolt/commit_config_test.go b/internal/storage/dolt/commit_config_test.go new file mode 100644 index 000000000..411bafe76 --- /dev/null +++ b/internal/storage/dolt/commit_config_test.go @@ -0,0 +1,502 @@ +package dolt + +import ( + "context" + "errors" + "regexp" + "slices" + "strings" + "testing" + + sqlmock "github.com/DATA-DOG/go-sqlmock" + + "github.com/steveyegge/beads/internal/storage" + "github.com/steveyegge/beads/internal/storage/issueops" +) + +// dirtyTables returns the set of table names currently in dolt_status. +func dirtyTables(t *testing.T, store *DoltStore) map[string]bool { + t.Helper() + ctx, cancel := testContext(t) + defer cancel() + + rows, err := store.db.QueryContext(ctx, "SELECT table_name FROM dolt_status") + if err != nil { + t.Fatalf("query dolt_status: %v", err) + } + defer rows.Close() + + tables := make(map[string]bool) + for rows.Next() { + var name string + if err := rows.Scan(&name); err != nil { + t.Fatalf("scan dolt_status: %v", err) + } + tables[name] = true + } + if err := rows.Err(); err != nil { + t.Fatalf("iterate dolt_status: %v", err) + } + return tables +} + +// GH#4078: a config-only working set must be committable. Generic Commit() +// excludes config by design (GH#2455), so intentional config writes +// (bd remember, bd config set) need a scoped commit that stages ONLY the +// config table and creates a real Dolt commit. +func TestCommitConfigCommitsConfigOnlyWorkingSet(t *testing.T) { + store, cleanup := setupTestStore(t) + defer cleanup() + + ctx, cancel := testContext(t) + defer cancel() + + // Start from a clean working set so the only dirty table is config. + if err := store.CommitWithConfig(ctx, "test: baseline"); err != nil { + t.Fatalf("baseline commit: %v", err) + } + before, err := store.GetCurrentCommit(ctx) + if err != nil { + t.Fatalf("get baseline commit: %v", err) + } + + if err := store.SetConfig(ctx, "test_gh4078_marker", "config-only write"); err != nil { + t.Fatalf("SetConfig: %v", err) + } + if !dirtyTables(t, store)["config"] { + t.Fatal("precondition failed: SetConfig did not leave config dirty") + } + + if err := store.CommitConfigOnly(ctx, "test: scoped config commit"); err != nil { + t.Fatalf("CommitConfigOnly: %v", err) + } + + after, err := store.GetCurrentCommit(ctx) + if err != nil { + t.Fatalf("get commit after CommitConfigOnly: %v", err) + } + if after == before { + t.Fatalf("CommitConfigOnly created no commit: HEAD still %s", before) + } + if dirtyTables(t, store)["config"] { + t.Fatal("config table still dirty after CommitConfigOnly") + } +} + +// GH#2455 must survive GH#4078: the scoped commit stages ONLY config. +// Dirty rows in other tables (e.g. a concurrent operation's half-written +// issue) must remain in the working set, not get swept into the commit. +func TestCommitConfigDoesNotSweepOtherDirtyTables(t *testing.T) { + store, cleanup := setupTestStore(t) + defer cleanup() + + ctx, cancel := testContext(t) + defer cancel() + + if err := store.CommitWithConfig(ctx, "test: baseline"); err != nil { + t.Fatalf("baseline commit: %v", err) + } + + // Dirty the issues table directly, bypassing any commit machinery — + // simulates a concurrent operation's uncommitted write. + if _, err := store.db.ExecContext(ctx, + "INSERT INTO issues (id, title, description, design, acceptance_criteria, notes, status, priority, issue_type)"+ + " VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", + "testdb-gh4078", "concurrent dirty row", "", "", "", "", "open", 2, "task"); err != nil { + t.Fatalf("dirty issues table: %v", err) + } + if err := store.SetConfig(ctx, "test_gh4078_scoped", "scoped"); err != nil { + t.Fatalf("SetConfig: %v", err) + } + + pre := dirtyTables(t, store) + if !pre["config"] || !pre["issues"] { + t.Fatalf("precondition failed: want config+issues dirty, got %v", pre) + } + + if err := store.CommitConfigOnly(ctx, "test: scoped config commit"); err != nil { + t.Fatalf("CommitConfigOnly: %v", err) + } + + post := dirtyTables(t, store) + if post["config"] { + t.Fatal("config table still dirty after CommitConfigOnly") + } + if !post["issues"] { + t.Fatal("CommitConfigOnly swept the dirty issues table — GH#2455 regression") + } +} + +// Pin the GH#2455 contract that motivates CommitConfigOnly's existence: the +// generic Commit() must keep EXCLUDING config. If this ever changes, the +// scoped-commit call sites should be revisited. +func TestCommitExcludesConfigOnlyWorkingSet(t *testing.T) { + store, cleanup := setupTestStore(t) + defer cleanup() + + ctx, cancel := testContext(t) + defer cancel() + + if err := store.CommitWithConfig(ctx, "test: baseline"); err != nil { + t.Fatalf("baseline commit: %v", err) + } + before, err := store.GetCurrentCommit(ctx) + if err != nil { + t.Fatalf("get baseline commit: %v", err) + } + + if err := store.SetConfig(ctx, "test_gh2455_excluded", "stays dirty"); err != nil { + t.Fatalf("SetConfig: %v", err) + } + + if err := store.Commit(ctx, "test: generic commit must skip config"); err != nil { + t.Fatalf("Commit: %v", err) + } + + after, err := store.GetCurrentCommit(ctx) + if err != nil { + t.Fatalf("get commit after Commit: %v", err) + } + if after != before { + t.Fatal("generic Commit() committed a config-only working set — GH#2455 regression") + } + if !dirtyTables(t, store)["config"] { + t.Fatal("config table no longer dirty after generic Commit()") + } +} + +// A memory write commits its own kv.* rows, and with them any other user kv.* +// rows already dirty: those are this clone's data, the same rows the pre-pull +// auto-commit would commit. +func TestCommitConfigUserKVOnlyCommitsUserKVRows(t *testing.T) { + store, cleanup := setupTestStore(t) + defer cleanup() + + ctx, cancel := testContext(t) + defer cancel() + + if err := store.CommitWithConfig(ctx, "test: baseline"); err != nil { + t.Fatalf("baseline commit: %v", err) + } + before, err := store.GetCurrentCommit(ctx) + if err != nil { + t.Fatalf("get baseline commit: %v", err) + } + + if err := store.SetConfig(ctx, "kv.memory.gh4078-a", "memory a"); err != nil { + t.Fatalf("SetConfig memory: %v", err) + } + if err := store.SetConfig(ctx, "kv.gh4078-flag", "user kv"); err != nil { + t.Fatalf("SetConfig kv: %v", err) + } + + if err := store.CommitConfigUserKVOnly(ctx, "test: memory commit"); err != nil { + t.Fatalf("CommitConfigUserKVOnly: %v", err) + } + + after, err := store.GetCurrentCommit(ctx) + if err != nil { + t.Fatalf("get commit after CommitConfigUserKVOnly: %v", err) + } + if after == before { + t.Fatalf("CommitConfigUserKVOnly created no commit: HEAD still %s", before) + } + if dirtyTables(t, store)["config"] { + t.Fatal("config table still dirty after CommitConfigUserKVOnly") + } +} + +// A dirty internal config key is not a memory write's to commit: the scoped +// commit refuses, names the key, and leaves HEAD and the working set alone. +func TestCommitConfigUserKVOnlyRefusesInternalConfigKeys(t *testing.T) { + store, cleanup := setupTestStore(t) + defer cleanup() + + ctx, cancel := testContext(t) + defer cancel() + + if err := store.CommitWithConfig(ctx, "test: baseline"); err != nil { + t.Fatalf("baseline commit: %v", err) + } + before, err := store.GetCurrentCommit(ctx) + if err != nil { + t.Fatalf("get baseline commit: %v", err) + } + + if err := store.SetConfig(ctx, "kv.memory.gh4078-b", "memory b"); err != nil { + t.Fatalf("SetConfig memory: %v", err) + } + if err := store.SetConfig(ctx, "test_gh4078_internal", "concurrent writer"); err != nil { + t.Fatalf("SetConfig internal: %v", err) + } + + err = store.CommitConfigUserKVOnly(ctx, "test: memory commit") + if err == nil { + t.Fatal("CommitConfigUserKVOnly committed over a dirty internal config key") + } + var refused *storage.ConfigCommitRefusedError + if !errors.As(err, &refused) { + t.Fatalf("error = %v, want a *storage.ConfigCommitRefusedError", err) + } + if !slices.Equal(refused.Keys, []string{"test_gh4078_internal"}) { + t.Fatalf("refused keys = %v, want [test_gh4078_internal]", refused.Keys) + } + if !strings.Contains(err.Error(), "test_gh4078_internal") { + t.Fatalf("error = %q, want it to name the internal key", err) + } + + assertRefusedCommitLeftNoTrace(t, store, before) +} + +// assertRefusedCommitLeftNoTrace checks a refused CommitConfigUserKVOnly's +// postconditions: HEAD is still before, nothing is staged for the next commit +// on the branch, and config is still dirty with the rows the refusal kept. +func assertRefusedCommitLeftNoTrace(t *testing.T, store *DoltStore, before string) { + t.Helper() + ctx, cancel := testContext(t) + defer cancel() + + after, err := store.GetCurrentCommit(ctx) + if err != nil { + t.Fatalf("get commit after refusal: %v", err) + } + if after != before { + t.Fatalf("HEAD moved from %s to %s on a refused commit", before, after) + } + staged, err := issueops.HasStagedChanges(ctx, store.db) + if err != nil { + t.Fatalf("check staged set after refusal: %v", err) + } + if staged { + t.Fatal("a refused commit left changes staged, so the next commit on the branch would publish them") + } + if !dirtyTables(t, store)["config"] { + t.Fatal("config table no longer dirty after a refused commit") + } +} + +// Sessions on one branch share its working set, so screening on the pinned +// connection that stages does not keep other sessions out: a key another +// session writes after the working-set screen is staged by DOLT_ADD all the +// same. The staged-set screen must refuse it and unstage config again. +func TestCommitConfigUserKVOnlyRefusesAKeyDirtiedAfterTheScreen(t *testing.T) { + // The second session writes while the commit holds its pinned + // connection, which setupTestStore's single connection would deadlock + // on. Not parallel: the test swaps the package-level seam. + store, cleanup := setupConcurrentTestStore(t) + defer cleanup() + + ctx, cancel := testContext(t) + defer cancel() + + if err := store.CommitWithConfig(ctx, "test: baseline"); err != nil { + t.Fatalf("baseline commit: %v", err) + } + before, err := store.GetCurrentCommit(ctx) + if err != nil { + t.Fatalf("get baseline commit: %v", err) + } + if err := store.SetConfig(ctx, "kv.memory.gh4078-race", "memory"); err != nil { + t.Fatalf("SetConfig memory: %v", err) + } + + var raceErr error + restore := afterStageCommitScreen + t.Cleanup(func() { afterStageCommitScreen = restore }) + afterStageCommitScreen = func() { + _, raceErr = store.db.ExecContext(ctx, + "INSERT INTO config (`key`, value) VALUES ('test_gh4078_race', 'concurrent writer')") + } + + err = store.CommitConfigUserKVOnly(ctx, "test: memory commit") + if raceErr != nil { + t.Fatalf("concurrent config write: %v", raceErr) + } + var refused *storage.ConfigCommitRefusedError + if !errors.As(err, &refused) { + t.Fatalf("CommitConfigUserKVOnly = %v, want a *storage.ConfigCommitRefusedError for the key written after the screen", err) + } + if !slices.Equal(refused.Keys, []string{"test_gh4078_race"}) { + t.Fatalf("refused keys = %v, want [test_gh4078_race]", refused.Keys) + } + + assertRefusedCommitLeftNoTrace(t, store, before) + var kept int + if err := store.db.QueryRowContext(ctx, + "SELECT COUNT(*) FROM config WHERE `key` IN ('kv.memory.gh4078-race', 'test_gh4078_race')").Scan(&kept); err != nil { + t.Fatalf("count working-set config rows: %v", err) + } + if kept != 2 { + t.Fatalf("%d of the 2 config rows are still in the working set after the refusal, want both", kept) + } +} + +// The refusal paths on the SQL they run: a key dirty before staging refuses +// before DOLT_ADD; one that only the staged set holds is unstaged with +// DOLT_RESET, and a failed unstage is reported alongside the refusal. None of +// them reaches DOLT_COMMIT. +func TestCommitConfigUserKVOnlyRefusalSQL(t *testing.T) { + const ( + workingDiff = "dolt_diff('HEAD', 'WORKING', 'config')" + stagedDiff = "dolt_diff('HEAD', 'STAGED', 'config')" + ) + resetFailure := errors.New("reset failed") + stageAndRefuse := func(mock sqlmock.Sqlmock) { + mock.ExpectQuery(regexp.QuoteMeta(workingDiff)). + WillReturnRows(sqlmock.NewRows([]string{"key"}).AddRow("kv.memory.a")) + mock.ExpectQuery(regexp.QuoteMeta("CALL DOLT_ADD(?)")). + WithArgs("config"). + WillReturnRows(sqlmock.NewRows([]string{"status"})) + mock.ExpectQuery(regexp.QuoteMeta(stagedDiff)). + WillReturnRows(sqlmock.NewRows([]string{"key"}).AddRow("kv.memory.a").AddRow("issue_prefix")) + } + for _, tc := range []struct { + name string + setup func(sqlmock.Sqlmock) + wantResetErr bool + }{ + { + name: "dirty before staging", + setup: func(mock sqlmock.Sqlmock) { + mock.ExpectQuery(regexp.QuoteMeta(workingDiff)). + WillReturnRows(sqlmock.NewRows([]string{"key"}).AddRow("kv.memory.a").AddRow("issue_prefix")) + }, + }, + { + name: "staged only", + setup: func(mock sqlmock.Sqlmock) { + stageAndRefuse(mock) + mock.ExpectQuery(regexp.QuoteMeta("CALL DOLT_RESET(?)")). + WithArgs("config"). + WillReturnRows(sqlmock.NewRows([]string{"status"})) + }, + }, + { + name: "staged only, unstage fails", + setup: func(mock sqlmock.Sqlmock) { + stageAndRefuse(mock) + mock.ExpectQuery(regexp.QuoteMeta("CALL DOLT_RESET(?)")). + WithArgs("config"). + WillReturnError(resetFailure) + }, + wantResetErr: true, + }, + } { + t.Run(tc.name, func(t *testing.T) { + db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherRegexp)) + if err != nil { + t.Fatalf("sqlmock: %v", err) + } + t.Cleanup(func() { _ = db.Close() }) + tc.setup(mock) + + store := &DoltStore{db: db} + err = store.CommitConfigUserKVOnly(context.Background(), "bd: remember") + + var refused *storage.ConfigCommitRefusedError + if !errors.As(err, &refused) { + t.Fatalf("CommitConfigUserKVOnly = %v, want a *storage.ConfigCommitRefusedError", err) + } + if !slices.Equal(refused.Keys, []string{"issue_prefix"}) { + t.Fatalf("refused keys = %v, want [issue_prefix]", refused.Keys) + } + if got := errors.Is(err, resetFailure); got != tc.wantResetErr { + t.Fatalf("error %q carries the reset failure = %v, want %v", err, got, tc.wantResetErr) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("unmet sqlmock expectations: %v", err) + } + }) + } +} + +// With versioned history active, every other fixed-list commit also stages the +// history tables, issues among them. A config write mints no version rows, so +// the scoped config commits stage only their own tables: widening them would +// sweep a concurrent issue mutation into a config or memory commit (GH#2455). +func TestScopedConfigCommitsSkipVersionedHistoryTables(t *testing.T) { + for _, tc := range []struct { + name string + commit func(*DoltStore, context.Context, string) error + }{ + {name: "CommitConfigOnly", commit: (*DoltStore).CommitConfigOnly}, + {name: "CommitConfigUserKVOnly", commit: (*DoltStore).CommitConfigUserKVOnly}, + } { + t.Run(tc.name, func(t *testing.T) { + store, cleanup := setupTestStore(t) + defer cleanup() + + ctx, cancel := testContext(t) + defer cancel() + + if err := store.CommitWithConfig(ctx, "test: baseline"); err != nil { + t.Fatalf("baseline commit: %v", err) + } + store.SetVersionedHistoryEnabled(true) + + if _, err := store.db.ExecContext(ctx, + "INSERT INTO issues (id, title, description, design, acceptance_criteria, notes, status, priority, issue_type)"+ + " VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", + "testdb-gh4078-history", "concurrent dirty row", "", "", "", "", "open", 2, "task"); err != nil { + t.Fatalf("dirty issues table: %v", err) + } + if err := store.SetConfig(ctx, "kv.gh4078-history", "user kv"); err != nil { + t.Fatalf("SetConfig: %v", err) + } + pre := dirtyTables(t, store) + if !pre["config"] || !pre["issues"] { + t.Fatalf("precondition failed: want config+issues dirty, got %v", pre) + } + + if err := tc.commit(store, ctx, "test: scoped config commit"); err != nil { + t.Fatalf("%s: %v", tc.name, err) + } + + post := dirtyTables(t, store) + if post["config"] { + t.Fatalf("config table still dirty after %s", tc.name) + } + if !post["issues"] { + t.Fatalf("%s swept the dirty issues table with versioned history on — GH#2455 regression", tc.name) + } + }) + } +} + +// SetConfig writes status.custom and types.custom into their lookup tables in +// the same transaction as the config row, so the scoped commit must take those +// tables with it: a commit holding the row without its projection would serve +// the old lookup set to every clone that pulls it. +func TestCommitConfigOnlyCommitsConfigProjections(t *testing.T) { + store, cleanup := setupTestStore(t) + defer cleanup() + + ctx, cancel := testContext(t) + defer cancel() + + if err := store.CommitWithConfig(ctx, "test: baseline"); err != nil { + t.Fatalf("baseline commit: %v", err) + } + + if err := store.SetConfig(ctx, "status.custom", "gh4078review:active"); err != nil { + t.Fatalf("SetConfig status.custom: %v", err) + } + if err := store.SetConfig(ctx, "types.custom", "gh4078kind"); err != nil { + t.Fatalf("SetConfig types.custom: %v", err) + } + pre := dirtyTables(t, store) + if !pre["config"] || !pre["custom_statuses"] || !pre["custom_types"] { + t.Fatalf("precondition failed: want config+custom_statuses+custom_types dirty, got %v", pre) + } + + if err := store.CommitConfigOnly(ctx, "test: scoped config commit"); err != nil { + t.Fatalf("CommitConfigOnly: %v", err) + } + + post := dirtyTables(t, store) + for _, table := range []string{"config", "custom_statuses", "custom_types"} { + if post[table] { + t.Fatalf("%s still dirty after CommitConfigOnly: got %v", table, post) + } + } +} diff --git a/internal/storage/dolt/store.go b/internal/storage/dolt/store.go index e04352b54..526414a22 100644 --- a/internal/storage/dolt/store.go +++ b/internal/storage/dolt/store.go @@ -1441,7 +1441,8 @@ func (s *DoltStore) scopeVersionedHistoryTransaction(tx *sql.Tx) func() { // writes to a fixed staging list when history is active on this store, so a // mutation's version rows land in that mutation's own Dolt commit rather than // sitting dirty in the working set. Every fixed-list DOLT_ADD path in this -// package routes through here; the tracker-based path uses +// package routes through here except the config-table commits, which mint no +// version rows (CommitConfigOnly); the tracker-based path uses // DirtyTableTracker.MarkVersionedHistoryDirty instead. // // Staging a table that turns out to be clean is free: DOLT_ADD stages nothing @@ -3775,26 +3776,10 @@ func (s *DoltStore) concludeOpenMerge(ctx context.Context, conn *sql.Conn, messa // config's primary key is `key`, so dolt_diff exposes to_key/from_key; an add or // delete leaves one side NULL, so COALESCE picks whichever key the change carries. func (s *DoltStore) assertDirtyConfigUserKVOnly(ctx context.Context, conn *sql.Conn) error { - rows, err := conn.QueryContext(ctx, - "SELECT COALESCE(to_key, from_key) FROM dolt_diff('HEAD', 'WORKING', 'config')") + unsafe, err := dirtyInternalConfigKeys(ctx, conn, false) if err != nil { return fmt.Errorf("inspect dirty config before pull: %w", err) } - defer rows.Close() - - var unsafe []string - for rows.Next() { - var key sql.NullString - if err := rows.Scan(&key); err != nil { - return fmt.Errorf("scan dirty config key: %w", err) - } - if key.Valid && !strings.HasPrefix(key.String, kvkeys.Prefix) { - unsafe = append(unsafe, key.String) - } - } - if err := rows.Err(); err != nil { - return fmt.Errorf("iterate dirty config diff: %w", err) - } if len(unsafe) > 0 { return fmt.Errorf("refusing to auto-commit %d dirty internal config key(s) before pull: %s; "+ "only user %s* keys auto-commit before a pull (GH#2455) — commit or revert "+ @@ -3803,6 +3788,38 @@ func (s *DoltStore) assertDirtyConfigUserKVOnly(ctx context.Context, conn *sql.C return nil } +// dirtyInternalConfigKeys lists the config keys that differ from HEAD and are +// not user KV data (outside the kv.* namespace): in the working set, or with +// staged set, in what DOLT_ADD has staged, which is what the next DOLT_COMMIT +// publishes. It is the screen shared by assertDirtyConfigUserKVOnly and +// CommitConfigUserKVOnly. +func dirtyInternalConfigKeys(ctx context.Context, conn *sql.Conn, staged bool) ([]string, error) { + query := "SELECT COALESCE(to_key, from_key) FROM dolt_diff('HEAD', 'WORKING', 'config')" + if staged { + query = "SELECT COALESCE(to_key, from_key) FROM dolt_diff('HEAD', 'STAGED', 'config')" + } + rows, err := conn.QueryContext(ctx, query) + if err != nil { + return nil, err + } + defer rows.Close() + + var unsafe []string + for rows.Next() { + var key sql.NullString + if err := rows.Scan(&key); err != nil { + return nil, fmt.Errorf("scan dirty config key: %w", err) + } + if key.Valid && !strings.HasPrefix(key.String, kvkeys.Prefix) { + unsafe = append(unsafe, key.String) + } + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("iterate dirty config diff: %w", err) + } + return unsafe, nil +} + // CommitWithConfig creates a Dolt commit that includes the config table. // Use this instead of Commit when the caller intentionally modified config // (e.g., CommitPending after 'bd config set', 'bd init', or 'bd rename-prefix'). @@ -3868,19 +3885,104 @@ func (s *DoltStore) CommitAll(ctx context.Context, message string) (bool, error) return committed, err } +// CommitConfigOnly commits the config table (GH#4078). Use after +// intentional config writes (bd config set) in server mode, +// where generic Commit() excludes config (GH#2455) and nothing else ever +// commits it. Scoped staging means a concurrent operation's dirty tables +// are never swept — the same guarantee that motivated GH#2455. +// +// The lookup tables SetConfig projects from config (custom_statuses, +// custom_types) are staged with it: SetConfig writes them in the same +// transaction as the row, and a commit holding one without the other is the +// split that projection exists to prevent. They are written only from config, +// so staging them sweeps no other kind of work. +// +// For the same reason the list is never widened with the versioned-history +// tables: a config write mints no version rows, and that list's `issues` would +// sweep a concurrent issue mutation into this commit. +func (s *DoltStore) CommitConfigOnly(ctx context.Context, message string) error { + tables := append([]string{"config"}, issueops.ConfigProjectionTables()...) + return s.stageAndCommit(ctx, tables, message, stageOptions{configPlane: true}) +} + +// CommitConfigUserKVOnly is the scoped commit for a write of this clone's own +// user KV data (bd remember, bd forget). It stages the config table alone, and +// since DOLT_ADD stages that whole table it applies the pre-pull screen: when +// any dirty config row is outside kv.*, it commits nothing and returns a +// *storage.ConfigCommitRefusedError naming those keys, rather than sweep a +// concurrent writer's internal key (issue_prefix above all) into a memory's +// commit — the row-level form of the GH#2455 hazard. +// +// The screen runs twice on the connection that stages and commits. Sessions on +// one branch share the working set, so a key another session writes after the +// first screen is still staged by DOLT_ADD; the second screen reads what was +// actually staged and, on a refusal, unstages config again (unstageRefused). +// What it cannot see is another commit path's own DOLT_ADD of config landing +// between the second screen and DOLT_COMMIT: that change is absorbed into this +// commit, the same absorption doltAddAndCommit documents for every scoped +// commit, and it is an intentional config commit, not a stale key. +func (s *DoltStore) CommitConfigUserKVOnly(ctx context.Context, message string) error { + return s.stageAndCommit(ctx, []string{"config"}, message, + stageOptions{configPlane: true, screen: refuseInternalConfigKeys}) +} + +// refuseInternalConfigKeys is CommitConfigUserKVOnly's screen: it fails with a +// *storage.ConfigCommitRefusedError when a config row outside kv.* differs from +// HEAD in the working set (before staging) or in the staged set (after). +func refuseInternalConfigKeys(ctx context.Context, conn *sql.Conn, staged bool) error { + unsafe, err := dirtyInternalConfigKeys(ctx, conn, staged) + if err != nil { + if staged { + return fmt.Errorf("inspect staged config: %w", err) + } + return fmt.Errorf("inspect dirty config: %w", err) + } + if len(unsafe) > 0 { + return &storage.ConfigCommitRefusedError{Keys: unsafe} + } + return nil +} + +// stageOptions are what the scoped commits ask of stageAndCommit beyond +// doltAddAndCommit's plain stage-and-commit. The zero value is +// doltAddAndCommit. +type stageOptions struct { + // configPlane stages the list exactly as given. Only the config-table + // commits set it (see CommitConfigOnly); every other list goes through + // withVersionedHistoryTables. + configPlane bool + // screen, when set, vets the commit on its own pinned connection: once + // against the working set before DOLT_ADD, and once against the staged + // set after it. A refusal before staging leaves nothing staged; a refusal + // after it unstages the tables again (unstageRefused). + screen func(ctx context.Context, conn *sql.Conn, staged bool) error +} + +// afterStageCommitScreen runs between stageAndCommit's working-set screen and +// its DOLT_ADD. It is a no-op outside tests, which use it to write in exactly +// the window the staged-set screen exists for. +var afterStageCommitScreen = func() {} + // doltAddAndCommit stages the specified tables and commits on a pinned // connection. This prevents DOLT_COMMIT('-Am') from sweeping up stale // working set changes from concurrent operations (GH#2455). Every caller has // already committed its SQL mutation, so any publication failure here has an // indeterminate durable outcome and must not be replayed. func (s *DoltStore) doltAddAndCommit(ctx context.Context, tables []string, commitMsg string) error { + return s.stageAndCommit(ctx, tables, commitMsg, stageOptions{}) +} + +// stageAndCommit is doltAddAndCommit with the scoped commits' options. +func (s *DoltStore) stageAndCommit(ctx context.Context, tables []string, commitMsg string, opts stageOptions) error { // Batch/off auto-commit (bd-4wamg): leave the writes in the working set // for a later explicit commit point (bd dolt commit / CommitPending), // matching doltAddAndCommitInTx. if issueops.VersionCommitDeferred(ctx) { return nil } - tables = s.withVersionedHistoryTables(tables) + if !opts.configPlane { + tables = s.withVersionedHistoryTables(tables) + } return s.withCircuitWrite(ctx, func(ctx context.Context) error { conn, err := s.db.Conn(ctx) if err != nil { @@ -3889,6 +3991,13 @@ func (s *DoltStore) doltAddAndCommit(ctx context.Context, tables []string, commi } defer conn.Close() + if opts.screen != nil { + if err := opts.screen(ctx, conn, false); err != nil { + return err + } + afterStageCommitScreen() + } + for _, table := range tables { if err := schema.DrainCall(ctx, conn, "CALL DOLT_ADD(?)", table); err != nil { return s.recordDoltPublicationFailure(ctx, @@ -3896,6 +4005,12 @@ func (s *DoltStore) doltAddAndCommit(ctx context.Context, tables []string, commi } } + if opts.screen != nil { + if err := opts.screen(ctx, conn, true); err != nil { + return unstageRefused(ctx, conn, tables, err) + } + } + // Skip the commit when nothing was actually staged (idempotent no-op // write), so Dolt does not log a server-side "nothing to commit" warning // on every reconcile-cadence call. The guard tests the STAGED set rather @@ -3934,6 +4049,25 @@ func (s *DoltStore) doltAddAndCommit(ctx context.Context, tables []string, commi }) } +// unstageRefused undoes stageAndCommit's DOLT_ADD after its screen refused the +// staged set, so the refused rows are not left staged for whichever commit +// runs next on the branch: every DOLT_COMMIT publishes the whole staged set. +// DOLT_RESET of a table only unstages it; its working-set changes stay. When +// the reset itself fails, both errors are returned. The reset is table-wide, +// so a concurrent config commit that staged config in the same window loses +// that staging too and publishes without those rows; they stay in the working +// set for the next config commit, which is the lesser hazard next to leaving +// a refused internal key staged for an unrelated commit to publish. +func unstageRefused(ctx context.Context, conn *sql.Conn, tables []string, refusal error) error { + for _, table := range tables { + if err := schema.DrainCall(ctx, conn, "CALL DOLT_RESET(?)", table); err != nil { + return fmt.Errorf("%w; unstaging %s afterwards also failed, so it stays staged for the next commit: %w", + refusal, table, err) + } + } + return refusal +} + func (s *DoltStore) wrapDoltPublicationFailure(ctx context.Context, op string, err error) error { return s.recordDoltPublicationFailure(ctx, wrapSQLCommitError(op, err)) } diff --git a/internal/storage/embeddeddolt/version_control.go b/internal/storage/embeddeddolt/version_control.go index 2afb2cc85..124ad798c 100644 --- a/internal/storage/embeddeddolt/version_control.go +++ b/internal/storage/embeddeddolt/version_control.go @@ -197,6 +197,21 @@ func (s *EmbeddedDoltStore) CommitMergeResolution(ctx context.Context, message s return err } +// CommitConfigOnly commits the config table (GH#4078). Embedded mode is +// single-process, so the '-Am' Commit cannot sweep a concurrent operation's +// working set — the scoped-staging concern that exists in server mode +// (GH#2455) does not apply here, and this aliases Commit for the same +// reason CommitWithConfig does. +func (s *EmbeddedDoltStore) CommitConfigOnly(ctx context.Context, message string) error { + return s.Commit(ctx, message) +} + +// CommitConfigUserKVOnly aliases Commit for the reason CommitConfigOnly does: +// with no concurrent writer, there is no other writer's config row to screen out. +func (s *EmbeddedDoltStore) CommitConfigUserKVOnly(ctx context.Context, message string) error { + return s.Commit(ctx, message) +} + func (s *EmbeddedDoltStore) AddRemote(ctx context.Context, name, url string) error { return s.AddRemoteWithRef(ctx, name, url, "") } diff --git a/internal/storage/issueops/config_helpers.go b/internal/storage/issueops/config_helpers.go index f17d858d4..bcef3bc4a 100644 --- a/internal/storage/issueops/config_helpers.go +++ b/internal/storage/issueops/config_helpers.go @@ -459,6 +459,7 @@ func ParseTypesConfigValue(value string) []string { // path must keep the projection in step with the string value or stale // table rows win forever. Returns the name of the synced table ("" when // the key has no projection) so transactional callers can mark it dirty. +// ConfigProjectionTables must list every table it can return. func SyncConfigTables(ctx context.Context, tx DBTX, key, value string) (string, error) { switch key { case "status.custom": @@ -475,6 +476,14 @@ func SyncConfigTables(ctx context.Context, tx DBTX, key, value string) (string, return "", nil } +// ConfigProjectionTables lists every table SyncConfigTables can write, for a +// caller that commits config writes without knowing which keys they touched: +// a config row committed without its projection is the split the projection +// exists to prevent. +func ConfigProjectionTables() []string { + return []string{"custom_statuses", "custom_types"} +} + // ResolveInfraTypesInTx reads infrastructure types from the database, // falling back to config.yaml then to hardcoded defaults. // Returns a map[string]bool for O(1) lookups. diff --git a/internal/storage/version_control.go b/internal/storage/version_control.go index baa79dda0..45830306e 100644 --- a/internal/storage/version_control.go +++ b/internal/storage/version_control.go @@ -2,7 +2,11 @@ package storage import ( "context" + "fmt" + "strings" "time" + + "github.com/steveyegge/beads/internal/storage/kvkeys" ) // CommitInfo represents a version control commit. @@ -163,7 +167,9 @@ type VersionControl interface { ListBranches(ctx context.Context) ([]string, error) Commit(ctx context.Context, message string) error // CommitWithConfig is like Commit but includes the config table. - // Use after intentional config writes (bd init, bd config set, bd rename-prefix). + // Use after intentional config writes (bd init, bd rename-prefix). Its '-Am' + // stages every dirty table, so a command committing only its own config + // write should use CommitConfigOnly or CommitConfigUserKVOnly instead. // GH#3216: bootstrap paths must use this to commit issue_prefix. CommitWithConfig(ctx context.Context, message string) error // CommitMergeResolution concludes a merge whose conflicts the operator @@ -190,6 +196,23 @@ type VersionControl interface { // racy HEAD-before/HEAD-after comparison (the interface threading tracked // as bd mybd-z9h7j; CommitPending already had the shape). CommitAll(ctx context.Context, message string) (bool, error) + // CommitConfigOnly commits the config table, with the lookup tables + // SetConfig projects from it (custom_statuses, custom_types) — the scoped + // commit for intentional config writes (bd config set) in server mode. + // There, unlike CommitWithConfig ('-Am'), it never stages other tables, so + // a concurrent operation's dirty working set is not swept (GH#2455). + // Embedded mode has no concurrent writer to sweep and commits the whole + // working set, as its CommitWithConfig does. + // GH#4078: without this, server-mode config writes strand the working + // set dirty — generic Commit() excludes config and silently no-ops. + CommitConfigOnly(ctx context.Context, message string) error + // CommitConfigUserKVOnly is the scoped commit for writes of user kv.* data + // (bd remember, bd forget). In server mode it stages the config table alone + // and refuses, committing nothing, when any dirty config row is outside + // kv.*, the screen the pre-pull auto-commit applies. The refusal is a + // *ConfigCommitRefusedError naming those keys. Embedded mode behaves as + // CommitConfigOnly. + CommitConfigUserKVOnly(ctx context.Context, message string) error CommitPending(ctx context.Context, actor string) (bool, error) CommitExists(ctx context.Context, commitHash string) (bool, error) GetCurrentCommit(ctx context.Context) (string, error) @@ -199,3 +222,21 @@ type VersionControl interface { GetConflicts(ctx context.Context) ([]Conflict, error) ResolveConflicts(ctx context.Context, table string, strategy string) error } + +// ConfigCommitRefusedError is CommitConfigUserKVOnly's refusal. Keys are the +// config rows outside the user kv.* namespace that were dirty, or that a +// concurrent writer dirtied while the commit staged: committing the user write +// would have swept them into it (GH#2455). Nothing was committed, and what the +// commit staged was unstaged again unless the error also reports that the +// unstage failed. The user write itself stays in the working set, so a caller +// whose write already landed reports the refusal rather than failing the write. +type ConfigCommitRefusedError struct { + Keys []string +} + +func (e *ConfigCommitRefusedError) Error() string { + return fmt.Sprintf("refusing to commit %d dirty internal config key(s) with a user %s* write: %s; "+ + "only user %s* rows commit with it (GH#2455). The %s* write itself is saved in the "+ + "working set: `bd dolt commit` publishes it together with those keys, or revert the "+ + "keys first", len(e.Keys), kvkeys.Prefix, strings.Join(e.Keys, ", "), kvkeys.Prefix, kvkeys.Prefix) +} diff --git a/tools/nogo/unvalidated_integration_tests.txt b/tools/nogo/unvalidated_integration_tests.txt index fce04a87e..acbbaeb56 100644 --- a/tools/nogo/unvalidated_integration_tests.txt +++ b/tools/nogo/unvalidated_integration_tests.txt @@ -16,6 +16,7 @@ # _test.go files). cmd/bd/ado_roundtrip_test.go cmd/bd/cli_fast_test.go +cmd/bd/config_commit_server_integration_test.go cmd/bd/contributor_routing_e2e_test.go cmd/bd/delete_test.go cmd/bd/doctor_health_integration_test.go